2024-11-13 18:15:37,778 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 18:15:37,836 main DEBUG Took 0.055581 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:15:37,837 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:15:37,837 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:15:37,839 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:15:37,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,861 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:15:37,902 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,906 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,908 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,909 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,910 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,913 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,915 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,916 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:15:37,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,918 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:15:37,920 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:15:37,921 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:15:37,923 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:15:37,924 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:15:37,929 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:15:37,929 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:15:37,963 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:15:37,971 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:15:37,974 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:15:37,977 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:15:37,977 main DEBUG createAppenders(={Console}) 2024-11-13 18:15:37,978 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 18:15:37,979 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 18:15:37,979 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 18:15:37,981 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:15:37,981 main DEBUG OutputStream closed 2024-11-13 18:15:37,982 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:15:37,982 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:15:37,982 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 18:15:38,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:15:38,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:15:38,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:15:38,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:15:38,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:15:38,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:15:38,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:15:38,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:15:38,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:15:38,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:15:38,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:15:38,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:15:38,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:15:38,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:15:38,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:15:38,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:15:38,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:15:38,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:15:38,106 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:15:38,107 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 18:15:38,107 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:15:38,113 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T18:15:38,140 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-13 18:15:38,144 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:15:38,144 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:15:38,189 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=691, ProcessCount=11, AvailableMemoryMB=2106 2024-11-13T18:15:38,484 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5 2024-11-13T18:15:38,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T18:15:38,506 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5, deleteOnExit=true 2024-11-13T18:15:38,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T18:15:38,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/test.cache.data in system properties and HBase conf 2024-11-13T18:15:38,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T18:15:38,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir in system properties and HBase conf 2024-11-13T18:15:38,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T18:15:38,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T18:15:38,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T18:15:38,631 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:15:38,761 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:15:38,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:15:38,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:15:38,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T18:15:38,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:15:38,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T18:15:38,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T18:15:38,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:15:38,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:15:38,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T18:15:38,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/nfs.dump.dir in system properties and HBase conf 2024-11-13T18:15:38,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/java.io.tmpdir in system properties and HBase conf 2024-11-13T18:15:38,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:15:38,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T18:15:38,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T18:15:39,836 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T18:15:39,983 INFO [Time-limited test {}] log.Log(170): Logging initialized @3073ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T18:15:40,088 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:15:40,167 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:15:40,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:15:40,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:15:40,245 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:15:40,267 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:15:40,274 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@667e8272{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:15:40,275 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3db55f04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:15:40,569 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@681e6b33{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/java.io.tmpdir/jetty-localhost-42429-hadoop-hdfs-3_4_1-tests_jar-_-any-12548670334503530162/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:15:40,583 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@66ea3674{HTTP/1.1, (http/1.1)}{localhost:42429} 2024-11-13T18:15:40,584 INFO [Time-limited test {}] server.Server(415): Started @3675ms 2024-11-13T18:15:41,097 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:15:41,104 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:15:41,106 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:15:41,106 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:15:41,106 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T18:15:41,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ee08c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:15:41,108 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71f8e357{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:15:41,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7937b685{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/java.io.tmpdir/jetty-localhost-33685-hadoop-hdfs-3_4_1-tests_jar-_-any-2880866652927331491/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:15:41,262 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15e7d6d5{HTTP/1.1, (http/1.1)}{localhost:33685} 2024-11-13T18:15:41,263 INFO [Time-limited test {}] server.Server(415): Started @4355ms 2024-11-13T18:15:41,334 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:15:41,506 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:15:41,513 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:15:41,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:15:41,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:15:41,516 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T18:15:41,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b86904d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:15:41,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5480bc23{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:15:41,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2925d9ef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/java.io.tmpdir/jetty-localhost-45533-hadoop-hdfs-3_4_1-tests_jar-_-any-13899001919115286391/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:15:41,686 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74214a4f{HTTP/1.1, (http/1.1)}{localhost:45533} 2024-11-13T18:15:41,687 INFO [Time-limited test {}] server.Server(415): Started @4778ms 2024-11-13T18:15:41,690 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:15:41,924 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:15:41,949 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:15:41,960 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:15:41,961 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:15:41,961 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:15:41,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f411fc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:15:41,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ebf56ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:15:42,021 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data2/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,021 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data4/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,021 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data3/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,021 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data1/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,247 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:15:42,250 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:15:42,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10fe3f07{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/java.io.tmpdir/jetty-localhost-40783-hadoop-hdfs-3_4_1-tests_jar-_-any-3640546453256956254/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:15:42,264 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5904946f{HTTP/1.1, (http/1.1)}{localhost:40783} 2024-11-13T18:15:42,265 INFO [Time-limited test {}] server.Server(415): Started @5356ms 2024-11-13T18:15:42,269 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:15:42,384 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16c01b9010e4f70 with lease ID 0xbd4a372347254194: Processing first storage report for DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8 from datanode DatanodeRegistration(127.0.0.1:38349, datanodeUuid=54b869a2-0ee7-4070-8387-5049c9b58854, infoPort=41085, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,385 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16c01b9010e4f70 with lease ID 0xbd4a372347254194: from storage DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8 node DatanodeRegistration(127.0.0.1:38349, datanodeUuid=54b869a2-0ee7-4070-8387-5049c9b58854, infoPort=41085, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3bc54ca0475204b5 with lease ID 0xbd4a372347254193: Processing first storage report for DS-2bb65705-56f0-4646-a633-f8a690c0d7ae from datanode DatanodeRegistration(127.0.0.1:37435, datanodeUuid=fcba98fd-d1a7-412b-b731-bd9fcad4325a, infoPort=32783, infoSecurePort=0, ipcPort=39715, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bc54ca0475204b5 with lease ID 0xbd4a372347254193: from storage DS-2bb65705-56f0-4646-a633-f8a690c0d7ae node DatanodeRegistration(127.0.0.1:37435, datanodeUuid=fcba98fd-d1a7-412b-b731-bd9fcad4325a, infoPort=32783, infoSecurePort=0, ipcPort=39715, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3bc54ca0475204b5 with lease ID 0xbd4a372347254193: Processing first storage report for DS-ebdff249-1abb-42c5-90ba-b6e16eeb5646 from datanode DatanodeRegistration(127.0.0.1:37435, datanodeUuid=fcba98fd-d1a7-412b-b731-bd9fcad4325a, infoPort=32783, infoSecurePort=0, ipcPort=39715, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3bc54ca0475204b5 with lease ID 0xbd4a372347254193: from storage DS-ebdff249-1abb-42c5-90ba-b6e16eeb5646 node DatanodeRegistration(127.0.0.1:37435, datanodeUuid=fcba98fd-d1a7-412b-b731-bd9fcad4325a, infoPort=32783, infoSecurePort=0, ipcPort=39715, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16c01b9010e4f70 with lease ID 0xbd4a372347254194: Processing first storage report for DS-4dc7c843-4486-45c2-8591-52889f815912 from datanode DatanodeRegistration(127.0.0.1:38349, datanodeUuid=54b869a2-0ee7-4070-8387-5049c9b58854, infoPort=41085, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16c01b9010e4f70 with lease ID 0xbd4a372347254194: from storage DS-4dc7c843-4486-45c2-8591-52889f815912 node DatanodeRegistration(127.0.0.1:38349, datanodeUuid=54b869a2-0ee7-4070-8387-5049c9b58854, infoPort=41085, infoSecurePort=0, ipcPort=43729, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,513 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data5/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,514 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data6/current/BP-1886344949-172.17.0.3-1731521739526/current, will proceed with Du for space computation calculation, 2024-11-13T18:15:42,615 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:15:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x369314cd67befd5c with lease ID 0xbd4a372347254195: Processing first storage report for DS-75a99b11-9703-4615-ac8d-dc275ad3ddeb from datanode DatanodeRegistration(127.0.0.1:34407, datanodeUuid=10001456-81e1-4ad3-9670-f5c1bebc8b50, infoPort=33047, infoSecurePort=0, ipcPort=42279, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x369314cd67befd5c with lease ID 0xbd4a372347254195: from storage DS-75a99b11-9703-4615-ac8d-dc275ad3ddeb node DatanodeRegistration(127.0.0.1:34407, datanodeUuid=10001456-81e1-4ad3-9670-f5c1bebc8b50, infoPort=33047, infoSecurePort=0, ipcPort=42279, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,625 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x369314cd67befd5c with lease ID 0xbd4a372347254195: Processing first storage report for DS-f5830a69-4f65-4adf-84d6-6065230f4e26 from datanode DatanodeRegistration(127.0.0.1:34407, datanodeUuid=10001456-81e1-4ad3-9670-f5c1bebc8b50, infoPort=33047, infoSecurePort=0, ipcPort=42279, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526) 2024-11-13T18:15:42,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x369314cd67befd5c with lease ID 0xbd4a372347254195: from storage DS-f5830a69-4f65-4adf-84d6-6065230f4e26 node DatanodeRegistration(127.0.0.1:34407, datanodeUuid=10001456-81e1-4ad3-9670-f5c1bebc8b50, infoPort=33047, infoSecurePort=0, ipcPort=42279, storageInfo=lv=-57;cid=testClusterID;nsid=2011204200;c=1731521739526), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:15:42,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5 2024-11-13T18:15:42,995 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/zookeeper_0, clientPort=55845, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:15:43,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55845 2024-11-13T18:15:43,029 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:43,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:15:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:15:43,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:15:43,842 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada with version=8 2024-11-13T18:15:43,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/hbase-staging 2024-11-13T18:15:43,978 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:15:44,323 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:15:44,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:44,341 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:44,350 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:15:44,350 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:44,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:15:44,556 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:15:44,640 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:15:44,650 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:15:44,654 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:15:44,682 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9717 (auto-detected) 2024-11-13T18:15:44,683 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:15:44,702 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41233 2024-11-13T18:15:44,724 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41233 connecting to ZooKeeper ensemble=127.0.0.1:55845 2024-11-13T18:15:44,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412330x0, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:15:44,774 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41233-0x1003a5b1ab20000 connected 2024-11-13T18:15:44,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:44,817 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:44,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:44,842 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada, hbase.cluster.distributed=false 2024-11-13T18:15:44,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:15:44,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41233 2024-11-13T18:15:44,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41233 2024-11-13T18:15:44,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41233 2024-11-13T18:15:44,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41233 2024-11-13T18:15:44,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41233 2024-11-13T18:15:45,032 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:15:45,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,034 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:15:45,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:15:45,037 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:15:45,040 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:15:45,041 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35435 2024-11-13T18:15:45,044 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35435 connecting to ZooKeeper ensemble=127.0.0.1:55845 2024-11-13T18:15:45,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,049 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354350x0, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:15:45,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:45,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35435-0x1003a5b1ab20001 connected 2024-11-13T18:15:45,062 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:15:45,069 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:15:45,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:15:45,077 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:15:45,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35435 2024-11-13T18:15:45,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35435 2024-11-13T18:15:45,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35435 2024-11-13T18:15:45,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35435 2024-11-13T18:15:45,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35435 2024-11-13T18:15:45,105 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:15:45,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,106 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:15:45,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:15:45,106 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:15:45,107 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:15:45,108 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46791 2024-11-13T18:15:45,110 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46791 connecting to ZooKeeper ensemble=127.0.0.1:55845 2024-11-13T18:15:45,111 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,113 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467910x0, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:15:45,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46791-0x1003a5b1ab20002 connected 2024-11-13T18:15:45,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:45,121 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:15:45,128 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:15:45,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:15:45,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:15:45,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-13T18:15:45,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46791 2024-11-13T18:15:45,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46791 2024-11-13T18:15:45,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-13T18:15:45,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-13T18:15:45,167 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:15:45,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,168 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:15:45,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:15:45,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:15:45,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:15:45,169 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:15:45,170 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42475 2024-11-13T18:15:45,172 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42475 connecting to ZooKeeper ensemble=127.0.0.1:55845 2024-11-13T18:15:45,174 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424750x0, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:15:45,187 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:15:45,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424750x0, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:15:45,193 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:15:45,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42475-0x1003a5b1ab20003 connected 2024-11-13T18:15:45,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:15:45,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:15:45,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42475 2024-11-13T18:15:45,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42475 2024-11-13T18:15:45,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42475 2024-11-13T18:15:45,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42475 2024-11-13T18:15:45,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42475 2024-11-13T18:15:45,220 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda96cf025dc:41233 2024-11-13T18:15:45,221 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda96cf025dc,41233,1731521744053 2024-11-13T18:15:45,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,232 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda96cf025dc,41233,1731521744053 2024-11-13T18:15:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:15:45,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:15:45,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:15:45,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,259 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:15:45,261 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda96cf025dc,41233,1731521744053 from backup master directory 2024-11-13T18:15:45,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda96cf025dc,41233,1731521744053 2024-11-13T18:15:45,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,267 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:15:45,267 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda96cf025dc,41233,1731521744053 2024-11-13T18:15:45,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:15:45,269 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:15:45,271 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:15:45,349 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/hbase.id] with ID: 7bfa8d2f-88be-48ac-b35b-b39c6e6cb643 2024-11-13T18:15:45,349 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/.tmp/hbase.id 2024-11-13T18:15:45,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:15:45,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:15:45,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:15:45,371 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/.tmp/hbase.id]:[hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/hbase.id] 2024-11-13T18:15:45,418 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:15:45,423 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:15:45,444 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-13T18:15:45,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:15:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:15:45,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:15:45,496 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:15:45,499 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:15:45,517 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:45,522 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:15:45,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:15:45,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:15:45,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:15:45,635 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store 2024-11-13T18:15:45,664 WARN [IPC Server handler 1 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:45,664 WARN [IPC Server handler 1 on default port 35347 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T18:15:45,664 WARN [IPC Server handler 1 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T18:15:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:15:45,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:15:45,686 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:15:45,690 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:45,691 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:15:45,692 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:45,692 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:45,694 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:15:45,694 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:45,694 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:15:45,696 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731521745691Disabling compacts and flushes for region at 1731521745691Disabling writes for close at 1731521745694 (+3 ms)Writing region close event to WAL at 1731521745694Closed at 1731521745694 2024-11-13T18:15:45,698 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/.initializing 2024-11-13T18:15:45,698 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/WALs/fda96cf025dc,41233,1731521744053 2024-11-13T18:15:45,710 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:15:45,737 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C41233%2C1731521744053, suffix=, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/WALs/fda96cf025dc,41233,1731521744053, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/oldWALs, maxLogs=10 2024-11-13T18:15:45,782 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/WALs/fda96cf025dc,41233,1731521744053/fda96cf025dc%2C41233%2C1731521744053.1731521745744, exclude list is [], retry=0 2024-11-13T18:15:45,791 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:45,791 WARN [IPC Server handler 2 on default port 35347 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T18:15:45,791 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T18:15:45,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:15:45,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:15:45,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T18:15:45,879 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/WALs/fda96cf025dc,41233,1731521744053/fda96cf025dc%2C41233%2C1731521744053.1731521745744 2024-11-13T18:15:45,884 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41085:41085),(127.0.0.1/127.0.0.1:32783:32783)] 2024-11-13T18:15:45,885 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:15:45,886 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:45,889 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:45,890 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:45,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:15:46,022 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:46,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:46,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:15:46,058 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:46,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:46,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:15:46,068 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:46,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:46,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:15:46,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:46,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:46,082 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,087 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,096 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,106 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,107 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,112 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:15:46,118 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:15:46,132 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:46,135 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72289780, jitterRate=0.07720166444778442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:15:46,145 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731521745909Initializing all the Stores at 1731521745915 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521745916 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521745920 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521745921 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521745921Cleaning up temporary data from old regions at 1731521746107 (+186 ms)Region opened successfully at 1731521746145 (+38 ms) 2024-11-13T18:15:46,147 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:15:46,214 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46676fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:15:46,270 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:15:46,286 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:15:46,286 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:15:46,290 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:15:46,294 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-13T18:15:46,302 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T18:15:46,302 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:15:46,365 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:15:46,377 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:15:46,381 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:15:46,384 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:15:46,386 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:15:46,390 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:15:46,393 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:15:46,398 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:15:46,405 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:15:46,413 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:15:46,415 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:15:46,443 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:15:46,445 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:15:46,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:46,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:46,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:46,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:15:46,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,461 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda96cf025dc,41233,1731521744053, sessionid=0x1003a5b1ab20000, setting cluster-up flag (Was=false) 2024-11-13T18:15:46,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,530 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:15:46,534 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,41233,1731521744053 2024-11-13T18:15:46,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:46,554 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:15:46,559 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,41233,1731521744053 2024-11-13T18:15:46,569 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:15:46,620 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(746): ClusterId : 7bfa8d2f-88be-48ac-b35b-b39c6e6cb643 2024-11-13T18:15:46,623 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:15:46,631 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(746): ClusterId : 7bfa8d2f-88be-48ac-b35b-b39c6e6cb643 2024-11-13T18:15:46,631 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:15:46,635 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:15:46,635 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:15:46,635 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:15:46,635 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:15:46,641 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:15:46,642 DEBUG [RS:0;fda96cf025dc:35435 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241ef7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:15:46,653 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(746): ClusterId : 7bfa8d2f-88be-48ac-b35b-b39c6e6cb643 2024-11-13T18:15:46,654 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:15:46,654 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:15:46,655 DEBUG [RS:1;fda96cf025dc:46791 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5c282f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:15:46,674 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:15:46,675 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:15:46,681 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:15:46,682 DEBUG [RS:2;fda96cf025dc:42475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b785542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:15:46,683 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda96cf025dc:35435 2024-11-13T18:15:46,687 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:15:46,687 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:15:46,687 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:15:46,691 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=35435, startcode=1731521744991 2024-11-13T18:15:46,691 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fda96cf025dc:46791 2024-11-13T18:15:46,691 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:15:46,692 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:15:46,692 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:15:46,710 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=46791, startcode=1731521745104 2024-11-13T18:15:46,711 DEBUG [RS:0;fda96cf025dc:35435 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:15:46,713 DEBUG [RS:1;fda96cf025dc:46791 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:15:46,714 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fda96cf025dc:42475 2024-11-13T18:15:46,715 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:15:46,715 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:15:46,715 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:15:46,717 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=42475, startcode=1731521745166 2024-11-13T18:15:46,717 DEBUG [RS:2;fda96cf025dc:42475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:15:46,768 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:15:46,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:15:46,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37855, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:15:46,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:15:46,798 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:15:46,798 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:46,807 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:15:46,820 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:46,821 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:46,815 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda96cf025dc,41233,1731521744053 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:15:46,831 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:15:46,831 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:15:46,831 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:15:46,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:15:46,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda96cf025dc:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:15:46,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:46,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:15:46,832 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:46,851 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521776851 2024-11-13T18:15:46,851 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:15:46,852 WARN [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:15:46,852 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:15:46,852 WARN [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:15:46,852 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:15:46,852 WARN [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:15:46,853 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:15:46,854 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:15:46,859 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:15:46,859 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:15:46,860 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:15:46,863 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:15:46,871 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:15:46,871 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:15:46,867 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:46,875 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:15:46,881 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:46,882 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:15:46,883 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:15:46,881 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:15:46,892 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:15:46,892 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:15:46,895 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521746893,5,FailOnTimeoutGroup] 2024-11-13T18:15:46,896 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521746895,5,FailOnTimeoutGroup] 2024-11-13T18:15:46,897 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:46,897 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:15:46,898 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:46,899 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:46,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:15:46,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:15:46,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:15:46,950 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:15:46,951 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:46,953 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=46791, startcode=1731521745104 2024-11-13T18:15:46,954 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=35435, startcode=1731521744991 2024-11-13T18:15:46,955 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,41233,1731521744053 with port=42475, startcode=1731521745166 2024-11-13T18:15:46,956 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,46791,1731521745104 2024-11-13T18:15:46,959 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:46,968 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,35435,1731521744991 2024-11-13T18:15:46,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:46,972 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:46,972 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35347 2024-11-13T18:15:46,972 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:15:46,976 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:46,976 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35347 2024-11-13T18:15:46,976 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:15:46,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,42475,1731521745166 2024-11-13T18:15:46,976 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:46,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:15:46,981 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:46,981 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:46,981 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T18:15:46,982 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T18:15:46,983 DEBUG [RS:1;fda96cf025dc:46791 {}] zookeeper.ZKUtil(111): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,46791,1731521745104 2024-11-13T18:15:46,983 WARN [RS:1;fda96cf025dc:46791 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:15:46,984 INFO [RS:1;fda96cf025dc:46791 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:15:46,984 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:46,984 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,46791,1731521745104 2024-11-13T18:15:46,984 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35347 2024-11-13T18:15:46,984 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:15:46,985 DEBUG [RS:0;fda96cf025dc:35435 {}] zookeeper.ZKUtil(111): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,35435,1731521744991 2024-11-13T18:15:46,985 WARN [RS:0;fda96cf025dc:35435 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:15:46,985 INFO [RS:0;fda96cf025dc:35435 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:15:46,985 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991 2024-11-13T18:15:46,988 DEBUG [RS:2;fda96cf025dc:42475 {}] zookeeper.ZKUtil(111): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,42475,1731521745166 2024-11-13T18:15:46,988 WARN [RS:2;fda96cf025dc:42475 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:15:46,988 INFO [RS:2;fda96cf025dc:42475 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:15:46,988 DEBUG [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166 2024-11-13T18:15:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:15:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:15:47,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:15:47,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:47,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,35435,1731521744991] 2024-11-13T18:15:47,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,46791,1731521745104] 2024-11-13T18:15:47,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,42475,1731521745166] 2024-11-13T18:15:47,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:15:47,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:15:47,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:15:47,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:15:47,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:15:47,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:15:47,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:15:47,081 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:15:47,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:15:47,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:15:47,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:15:47,099 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:15:47,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:15:47,109 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:15:47,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:15:47,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:15:47,113 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:15:47,116 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:15:47,117 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:15:47,118 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:15:47,129 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:15:47,139 INFO [RS:2;fda96cf025dc:42475 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:15:47,139 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,140 INFO [RS:0;fda96cf025dc:35435 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:15:47,140 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,141 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:15:47,143 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:15:47,143 INFO [RS:1;fda96cf025dc:46791 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:15:47,143 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,144 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:15:47,147 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:47,148 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64269146, jitterRate=-0.04231509566307068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:15:47,149 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:15:47,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731521747008Initializing all the Stores at 1731521747010 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747010Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747031 (+21 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521747032 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747032Cleaning up temporary data from old regions at 1731521747111 (+79 ms)Region opened successfully at 1731521747149 (+38 ms) 2024-11-13T18:15:47,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:15:47,150 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:15:47,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:15:47,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:15:47,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:15:47,151 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,151 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:15:47,151 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,151 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,152 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,152 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,152 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:15:47,152 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,152 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,152 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,152 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:15:47,152 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,153 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:0;fda96cf025dc:35435 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,154 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:2;fda96cf025dc:42475 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,154 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,154 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,155 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:15:47,155 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,155 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,156 DEBUG [RS:1;fda96cf025dc:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:15:47,157 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:15:47,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731521747150Disabling compacts and flushes for region at 1731521747150Disabling writes for close at 1731521747150Writing region close event to WAL at 1731521747157 (+7 ms)Closed at 1731521747157 2024-11-13T18:15:47,160 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,161 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,161 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,161 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,161 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,161 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,35435,1731521744991-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:15:47,166 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:15:47,166 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:15:47,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:15:47,186 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,187 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,187 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,187 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,187 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,187 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46791,1731521745104-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:15:47,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:15:47,196 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:15:47,196 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:15:47,198 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,35435,1731521744991-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,199 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,199 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.Replication(171): fda96cf025dc,35435,1731521744991 started 2024-11-13T18:15:47,213 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,213 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,213 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,213 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,214 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,214 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42475,1731521745166-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:15:47,228 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:15:47,229 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,46791,1731521745104-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,231 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,231 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.Replication(171): fda96cf025dc,46791,1731521745104 started 2024-11-13T18:15:47,236 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,236 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,35435,1731521744991, RpcServer on fda96cf025dc/172.17.0.3:35435, sessionid=0x1003a5b1ab20001 2024-11-13T18:15:47,237 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:15:47,238 DEBUG [RS:0;fda96cf025dc:35435 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,35435,1731521744991 2024-11-13T18:15:47,238 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,35435,1731521744991' 2024-11-13T18:15:47,238 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:15:47,239 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:15:47,239 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42475,1731521745166-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,239 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,240 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.Replication(171): fda96cf025dc,42475,1731521745166 started 2024-11-13T18:15:47,241 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:15:47,245 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:15:47,245 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:15:47,245 DEBUG [RS:0;fda96cf025dc:35435 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,35435,1731521744991 2024-11-13T18:15:47,246 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,35435,1731521744991' 2024-11-13T18:15:47,246 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:15:47,249 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:15:47,250 DEBUG [RS:0;fda96cf025dc:35435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:15:47,250 INFO [RS:0;fda96cf025dc:35435 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:15:47,250 INFO [RS:0;fda96cf025dc:35435 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:15:47,261 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,261 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,46791,1731521745104, RpcServer on fda96cf025dc/172.17.0.3:46791, sessionid=0x1003a5b1ab20002 2024-11-13T18:15:47,261 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:15:47,261 DEBUG [RS:1;fda96cf025dc:46791 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,46791,1731521745104 2024-11-13T18:15:47,261 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,46791,1731521745104' 2024-11-13T18:15:47,261 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:15:47,262 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:15:47,263 INFO [RS:2;fda96cf025dc:42475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:47,264 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,42475,1731521745166, RpcServer on fda96cf025dc/172.17.0.3:42475, sessionid=0x1003a5b1ab20003 2024-11-13T18:15:47,264 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:15:47,264 DEBUG [RS:2;fda96cf025dc:42475 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,42475,1731521745166 2024-11-13T18:15:47,264 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,42475,1731521745166' 2024-11-13T18:15:47,264 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:15:47,265 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:15:47,265 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:15:47,265 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:15:47,266 DEBUG [RS:1;fda96cf025dc:46791 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,46791,1731521745104 2024-11-13T18:15:47,266 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,46791,1731521745104' 2024-11-13T18:15:47,266 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:15:47,272 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:15:47,273 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:15:47,273 DEBUG [RS:2;fda96cf025dc:42475 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,42475,1731521745166 2024-11-13T18:15:47,273 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,42475,1731521745166' 2024-11-13T18:15:47,273 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:15:47,273 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:15:47,277 DEBUG [RS:1;fda96cf025dc:46791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:15:47,277 INFO [RS:1;fda96cf025dc:46791 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:15:47,277 INFO [RS:1;fda96cf025dc:46791 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:15:47,280 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:15:47,281 DEBUG [RS:2;fda96cf025dc:42475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:15:47,281 INFO [RS:2;fda96cf025dc:42475 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:15:47,281 INFO [RS:2;fda96cf025dc:42475 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:15:47,347 WARN [fda96cf025dc:41233 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:15:47,356 INFO [RS:0;fda96cf025dc:35435 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:15:47,363 INFO [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C35435%2C1731521744991, suffix=, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs, maxLogs=32 2024-11-13T18:15:47,378 INFO [RS:1;fda96cf025dc:46791 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:15:47,382 INFO [RS:2;fda96cf025dc:42475 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:15:47,386 DEBUG [RS:0;fda96cf025dc:35435 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991/fda96cf025dc%2C35435%2C1731521744991.1731521747365, exclude list is [], retry=0 2024-11-13T18:15:47,387 INFO [RS:1;fda96cf025dc:46791 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C46791%2C1731521745104, suffix=, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,46791,1731521745104, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs, maxLogs=32 2024-11-13T18:15:47,389 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:47,389 WARN [IPC Server handler 2 on default port 35347 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T18:15:47,389 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T18:15:47,393 INFO [RS:2;fda96cf025dc:42475 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C42475%2C1731521745166, suffix=, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs, maxLogs=32 2024-11-13T18:15:47,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:15:47,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:15:47,414 DEBUG [RS:1;fda96cf025dc:46791 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,46791,1731521745104/fda96cf025dc%2C46791%2C1731521745104.1731521747390, exclude list is [], retry=0 2024-11-13T18:15:47,418 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T18:15:47,418 WARN [IPC Server handler 2 on default port 35347 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T18:15:47,418 WARN [IPC Server handler 2 on default port 35347 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T18:15:47,421 INFO [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991/fda96cf025dc%2C35435%2C1731521744991.1731521747365 2024-11-13T18:15:47,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:15:47,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:15:47,426 DEBUG [RS:2;fda96cf025dc:42475 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166/fda96cf025dc%2C42475%2C1731521745166.1731521747395, exclude list is [], retry=0 2024-11-13T18:15:47,429 DEBUG [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41085:41085),(127.0.0.1/127.0.0.1:32783:32783)] 2024-11-13T18:15:47,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34407,DS-75a99b11-9703-4615-ac8d-dc275ad3ddeb,DISK] 2024-11-13T18:15:47,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:15:47,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:15:47,462 INFO [RS:1;fda96cf025dc:46791 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,46791,1731521745104/fda96cf025dc%2C46791%2C1731521745104.1731521747390 2024-11-13T18:15:47,463 DEBUG [RS:1;fda96cf025dc:46791 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41085:41085),(127.0.0.1/127.0.0.1:32783:32783)] 2024-11-13T18:15:47,464 INFO [RS:2;fda96cf025dc:42475 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166/fda96cf025dc%2C42475%2C1731521745166.1731521747395 2024-11-13T18:15:47,465 DEBUG [RS:2;fda96cf025dc:42475 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33047:33047),(127.0.0.1/127.0.0.1:32783:32783),(127.0.0.1/127.0.0.1:41085:41085)] 2024-11-13T18:15:47,600 DEBUG [fda96cf025dc:41233 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T18:15:47,614 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:15:47,624 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T18:15:47,624 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:15:47,625 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:15:47,625 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T18:15:47,625 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:15:47,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:47,645 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPENING 2024-11-13T18:15:47,660 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:15:47,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:47,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:47,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:47,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:15:47,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:47,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:47,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:47,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:47,682 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:15:47,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:47,865 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:15:47,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36501, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:15:47,883 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:15:47,884 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:15:47,885 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:15:47,888 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C42475%2C1731521745166.meta, suffix=.meta, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs, maxLogs=32 2024-11-13T18:15:47,909 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, exclude list is [], retry=0 2024-11-13T18:15:47,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:15:47,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:15:47,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34407,DS-75a99b11-9703-4615-ac8d-dc275ad3ddeb,DISK] 2024-11-13T18:15:47,946 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta 2024-11-13T18:15:47,947 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33047:33047),(127.0.0.1/127.0.0.1:32783:32783),(127.0.0.1/127.0.0.1:41085:41085)] 2024-11-13T18:15:47,947 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:15:47,949 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:15:47,952 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:15:47,957 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:15:47,962 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:15:47,962 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:47,963 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:15:47,963 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:15:47,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:15:47,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:15:47,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:15:47,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:15:47,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:15:47,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:15:47,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:15:47,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:15:47,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:47,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:15:47,981 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:15:47,983 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:15:47,988 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:15:47,995 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:15:47,995 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:15:47,996 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:15:48,000 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:15:48,002 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62180531, jitterRate=-0.0734378844499588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:15:48,002 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:15:48,004 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731521747963Writing region info on filesystem at 1731521747964 (+1 ms)Initializing all the Stores at 1731521747966 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747966Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747966Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521747966Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521747966Cleaning up temporary data from old regions at 1731521747995 (+29 ms)Running coprocessor post-open hooks at 1731521748002 (+7 ms)Region opened successfully at 1731521748004 (+2 ms) 2024-11-13T18:15:48,015 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731521747855 2024-11-13T18:15:48,030 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:15:48,030 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:15:48,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:48,035 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPEN 2024-11-13T18:15:48,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:48,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:48,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:48,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:48,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:48,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:48,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:48,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:48,038 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:48,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:15:48,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,42475,1731521745166 in 355 msec 2024-11-13T18:15:48,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:15:48,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 873 msec 2024-11-13T18:15:48,055 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:15:48,055 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:15:48,088 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:48,090 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:48,142 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:15:48,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46819, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:15:48,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5410 sec 2024-11-13T18:15:48,191 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731521748191, completionTime=-1 2024-11-13T18:15:48,195 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T18:15:48,195 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:15:48,256 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T18:15:48,257 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521808256 2024-11-13T18:15:48,257 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731521868257 2024-11-13T18:15:48,257 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 61 msec 2024-11-13T18:15:48,259 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:15:48,267 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,268 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,268 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,270 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda96cf025dc:41233, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,277 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,279 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:15:48,322 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.055sec 2024-11-13T18:15:48,324 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:15:48,326 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:15:48,327 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:15:48,328 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:15:48,328 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:15:48,329 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:15:48,330 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:15:48,345 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:15:48,346 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:15:48,346 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41233,1731521744053-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:15:48,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6906c3d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:15:48,361 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:15:48,361 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:15:48,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,41233,-1 for getting cluster id 2024-11-13T18:15:48,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:15:48,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7bfa8d2f-88be-48ac-b35b-b39c6e6cb643' 2024-11-13T18:15:48,394 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:15:48,394 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7bfa8d2f-88be-48ac-b35b-b39c6e6cb643" 2024-11-13T18:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574d550e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,41233,-1] 2024-11-13T18:15:48,399 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:15:48,402 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:48,407 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:15:48,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c420a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:15:48,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:48,421 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:48,422 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:15:48,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:15:48,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda96cf025dc,41233,1731521744053 2024-11-13T18:15:48,457 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T18:15:48,465 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:15:48,471 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,41233,1731521744053 2024-11-13T18:15:48,476 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@154fa0a8 2024-11-13T18:15:48,478 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:15:48,488 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:15:48,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:15:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:15:48,580 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:15:48,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-13T18:15:48,585 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:48,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:15:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:48,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741837_1013 (size=1020) 2024-11-13T18:15:48,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741837_1013 (size=1020) 2024-11-13T18:15:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741837_1013 (size=1020) 2024-11-13T18:15:48,659 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd437c88e4af73574ce0ce0882aca142, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,661 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 38c892b7353daa24a9b6c917f1aa0a89, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,661 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 02893942b5e22ded04197e4e8486e0dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:48,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741838_1014 (size=107) 2024-11-13T18:15:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741839_1015 (size=110) 2024-11-13T18:15:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741838_1014 (size=107) 2024-11-13T18:15:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741838_1014 (size=107) 2024-11-13T18:15:48,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741839_1015 (size=110) 2024-11-13T18:15:48,717 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,717 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 38c892b7353daa24a9b6c917f1aa0a89, disabling compactions & flushes 2024-11-13T18:15:48,717 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:48,717 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:48,718 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. after waiting 0 ms 2024-11-13T18:15:48,718 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:48,718 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:48,718 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 38c892b7353daa24a9b6c917f1aa0a89: Waiting for close lock at 1731521748717Disabling compacts and flushes for region at 1731521748717Disabling writes for close at 1731521748718 (+1 ms)Writing region close event to WAL at 1731521748718Closed at 1731521748718 2024-11-13T18:15:48,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741839_1015 (size=110) 2024-11-13T18:15:48,719 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 00a4d576f4c41497f18224865b73403a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,719 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,720 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing fd437c88e4af73574ce0ce0882aca142, disabling compactions & flushes 2024-11-13T18:15:48,720 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:48,720 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:48,720 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. after waiting 0 ms 2024-11-13T18:15:48,720 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:48,720 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:48,720 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd437c88e4af73574ce0ce0882aca142: Waiting for close lock at 1731521748719Disabling compacts and flushes for region at 1731521748719Disabling writes for close at 1731521748720 (+1 ms)Writing region close event to WAL at 1731521748720Closed at 1731521748720 2024-11-13T18:15:48,721 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e715c3850b0001604b62543d4e17908, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741840_1016 (size=110) 2024-11-13T18:15:48,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741840_1016 (size=110) 2024-11-13T18:15:48,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741840_1016 (size=110) 2024-11-13T18:15:48,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,736 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 02893942b5e22ded04197e4e8486e0dc, disabling compactions & flushes 2024-11-13T18:15:48,736 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:48,736 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:48,736 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. after waiting 0 ms 2024-11-13T18:15:48,736 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:48,736 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:48,736 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 02893942b5e22ded04197e4e8486e0dc: Waiting for close lock at 1731521748736Disabling compacts and flushes for region at 1731521748736Disabling writes for close at 1731521748736Writing region close event to WAL at 1731521748736Closed at 1731521748736 2024-11-13T18:15:48,737 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 09ff4ab7a9d0091782b93c194b270c43, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741841_1017 (size=110) 2024-11-13T18:15:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741841_1017 (size=110) 2024-11-13T18:15:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741841_1017 (size=110) 2024-11-13T18:15:48,762 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,762 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 00a4d576f4c41497f18224865b73403a, disabling compactions & flushes 2024-11-13T18:15:48,762 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:48,762 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:48,762 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. after waiting 0 ms 2024-11-13T18:15:48,762 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:48,763 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:48,763 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 00a4d576f4c41497f18224865b73403a: Waiting for close lock at 1731521748762Disabling compacts and flushes for region at 1731521748762Disabling writes for close at 1731521748762Writing region close event to WAL at 1731521748763 (+1 ms)Closed at 1731521748763 2024-11-13T18:15:48,763 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 801d0f207e45efea50135a16f5fcbc99, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741842_1018 (size=110) 2024-11-13T18:15:48,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741842_1018 (size=110) 2024-11-13T18:15:48,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741842_1018 (size=110) 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 1e715c3850b0001604b62543d4e17908, disabling compactions & flushes 2024-11-13T18:15:48,784 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. after waiting 0 ms 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:48,784 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:48,784 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e715c3850b0001604b62543d4e17908: Waiting for close lock at 1731521748784Disabling compacts and flushes for region at 1731521748784Disabling writes for close at 1731521748784Writing region close event to WAL at 1731521748784Closed at 1731521748784 2024-11-13T18:15:48,785 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c39d6007aedf97e4c4a94511fe061ef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741843_1019 (size=110) 2024-11-13T18:15:48,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741843_1019 (size=110) 2024-11-13T18:15:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741843_1019 (size=110) 2024-11-13T18:15:48,790 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,790 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 09ff4ab7a9d0091782b93c194b270c43, disabling compactions & flushes 2024-11-13T18:15:48,790 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:48,790 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:48,790 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. after waiting 0 ms 2024-11-13T18:15:48,790 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:48,791 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:48,791 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 09ff4ab7a9d0091782b93c194b270c43: Waiting for close lock at 1731521748790Disabling compacts and flushes for region at 1731521748790Disabling writes for close at 1731521748790Writing region close event to WAL at 1731521748790Closed at 1731521748790 2024-11-13T18:15:48,791 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7fb2e398d735ceeb281622c9b30d4cf1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741844_1020 (size=110) 2024-11-13T18:15:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741844_1020 (size=110) 2024-11-13T18:15:48,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741844_1020 (size=110) 2024-11-13T18:15:48,821 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,821 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 801d0f207e45efea50135a16f5fcbc99, disabling compactions & flushes 2024-11-13T18:15:48,821 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:48,821 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:48,821 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. after waiting 0 ms 2024-11-13T18:15:48,821 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:48,821 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:48,822 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 801d0f207e45efea50135a16f5fcbc99: Waiting for close lock at 1731521748821Disabling compacts and flushes for region at 1731521748821Disabling writes for close at 1731521748821Writing region close event to WAL at 1731521748821Closed at 1731521748821 2024-11-13T18:15:48,823 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5aa68e8ac635bc9e7d6868f5f771281e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741845_1021 (size=110) 2024-11-13T18:15:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741845_1021 (size=110) 2024-11-13T18:15:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741845_1021 (size=110) 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 5c39d6007aedf97e4c4a94511fe061ef, disabling compactions & flushes 2024-11-13T18:15:48,841 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. after waiting 0 ms 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:48,841 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:48,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c39d6007aedf97e4c4a94511fe061ef: Waiting for close lock at 1731521748841Disabling compacts and flushes for region at 1731521748841Disabling writes for close at 1731521748841Writing region close event to WAL at 1731521748841Closed at 1731521748841 2024-11-13T18:15:48,842 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5bab85a9454bcf58ade3574335c3bd1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741846_1022 (size=110) 2024-11-13T18:15:48,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741846_1022 (size=110) 2024-11-13T18:15:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741846_1022 (size=110) 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 7fb2e398d735ceeb281622c9b30d4cf1, disabling compactions & flushes 2024-11-13T18:15:48,856 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. after waiting 0 ms 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:48,856 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:48,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7fb2e398d735ceeb281622c9b30d4cf1: Waiting for close lock at 1731521748856Disabling compacts and flushes for region at 1731521748856Disabling writes for close at 1731521748856Writing region close event to WAL at 1731521748856Closed at 1731521748856 2024-11-13T18:15:48,857 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8636f69311e6a46aa80948ea8be00961, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741848_1024 (size=110) 2024-11-13T18:15:48,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741848_1024 (size=110) 2024-11-13T18:15:48,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741848_1024 (size=110) 2024-11-13T18:15:48,882 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,882 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing e5bab85a9454bcf58ade3574335c3bd1, disabling compactions & flushes 2024-11-13T18:15:48,882 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:48,882 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:48,882 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. after waiting 0 ms 2024-11-13T18:15:48,882 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:48,882 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:48,883 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5bab85a9454bcf58ade3574335c3bd1: Waiting for close lock at 1731521748882Disabling compacts and flushes for region at 1731521748882Disabling writes for close at 1731521748882Writing region close event to WAL at 1731521748882Closed at 1731521748882 2024-11-13T18:15:48,884 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 27ae2919589ed9f92a8ad04abb8e24d5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741847_1023 (size=110) 2024-11-13T18:15:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741847_1023 (size=110) 2024-11-13T18:15:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741847_1023 (size=110) 2024-11-13T18:15:48,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 5aa68e8ac635bc9e7d6868f5f771281e, disabling compactions & flushes 2024-11-13T18:15:48,890 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:48,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:48,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. after waiting 0 ms 2024-11-13T18:15:48,891 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:48,891 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:48,891 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5aa68e8ac635bc9e7d6868f5f771281e: Waiting for close lock at 1731521748890Disabling compacts and flushes for region at 1731521748890Disabling writes for close at 1731521748890Writing region close event to WAL at 1731521748891 (+1 ms)Closed at 1731521748891 2024-11-13T18:15:48,892 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 37863ce38db650b86b62a2bccc5e7796, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741849_1025 (size=110) 2024-11-13T18:15:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741849_1025 (size=110) 2024-11-13T18:15:48,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741849_1025 (size=110) 2024-11-13T18:15:48,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 8636f69311e6a46aa80948ea8be00961, disabling compactions & flushes 2024-11-13T18:15:48,935 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:48,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:48,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. after waiting 0 ms 2024-11-13T18:15:48,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:48,935 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:48,936 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8636f69311e6a46aa80948ea8be00961: Waiting for close lock at 1731521748935Disabling compacts and flushes for region at 1731521748935Disabling writes for close at 1731521748935Writing region close event to WAL at 1731521748935Closed at 1731521748935 2024-11-13T18:15:48,937 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3eefcb355d011c4ea65b0572629d5ef5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741850_1026 (size=110) 2024-11-13T18:15:48,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741850_1026 (size=110) 2024-11-13T18:15:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741850_1026 (size=110) 2024-11-13T18:15:48,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 27ae2919589ed9f92a8ad04abb8e24d5, disabling compactions & flushes 2024-11-13T18:15:48,957 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:48,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:48,958 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. after waiting 0 ms 2024-11-13T18:15:48,958 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:48,958 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:48,958 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 27ae2919589ed9f92a8ad04abb8e24d5: Waiting for close lock at 1731521748957Disabling compacts and flushes for region at 1731521748957Disabling writes for close at 1731521748958 (+1 ms)Writing region close event to WAL at 1731521748958Closed at 1731521748958 2024-11-13T18:15:48,959 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 575404eded2e7a15cbfa9186eea1d1b1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741851_1027 (size=110) 2024-11-13T18:15:48,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741851_1027 (size=110) 2024-11-13T18:15:48,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741851_1027 (size=110) 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 37863ce38db650b86b62a2bccc5e7796, disabling compactions & flushes 2024-11-13T18:15:48,966 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. after waiting 0 ms 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:48,966 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:48,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 37863ce38db650b86b62a2bccc5e7796: Waiting for close lock at 1731521748966Disabling compacts and flushes for region at 1731521748966Disabling writes for close at 1731521748966Writing region close event to WAL at 1731521748966Closed at 1731521748966 2024-11-13T18:15:48,967 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dae4cf552a1afaf0b041668499d99e20, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741852_1028 (size=110) 2024-11-13T18:15:48,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741852_1028 (size=110) 2024-11-13T18:15:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741852_1028 (size=110) 2024-11-13T18:15:48,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 3eefcb355d011c4ea65b0572629d5ef5, disabling compactions & flushes 2024-11-13T18:15:48,983 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:48,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:48,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. after waiting 0 ms 2024-11-13T18:15:48,984 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:48,984 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:48,984 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3eefcb355d011c4ea65b0572629d5ef5: Waiting for close lock at 1731521748983Disabling compacts and flushes for region at 1731521748983Disabling writes for close at 1731521748983Writing region close event to WAL at 1731521748984 (+1 ms)Closed at 1731521748984 2024-11-13T18:15:48,985 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 64b5086eb7020ccba10769de2eaf3d9b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741854_1030 (size=110) 2024-11-13T18:15:48,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741854_1030 (size=110) 2024-11-13T18:15:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741854_1030 (size=110) 2024-11-13T18:15:48,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:48,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing dae4cf552a1afaf0b041668499d99e20, disabling compactions & flushes 2024-11-13T18:15:48,992 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:48,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:48,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. after waiting 0 ms 2024-11-13T18:15:48,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:48,993 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:48,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for dae4cf552a1afaf0b041668499d99e20: Waiting for close lock at 1731521748992Disabling compacts and flushes for region at 1731521748992Disabling writes for close at 1731521748993 (+1 ms)Writing region close event to WAL at 1731521748993Closed at 1731521748993 2024-11-13T18:15:48,994 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1b91248305f9bd170badfc1ac16c2f2b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:48,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741853_1029 (size=110) 2024-11-13T18:15:49,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741853_1029 (size=110) 2024-11-13T18:15:49,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741853_1029 (size=110) 2024-11-13T18:15:49,016 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,016 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 575404eded2e7a15cbfa9186eea1d1b1, disabling compactions & flushes 2024-11-13T18:15:49,016 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. after waiting 0 ms 2024-11-13T18:15:49,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,017 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 575404eded2e7a15cbfa9186eea1d1b1: Waiting for close lock at 1731521749016Disabling compacts and flushes for region at 1731521749016Disabling writes for close at 1731521749017 (+1 ms)Writing region close event to WAL at 1731521749017Closed at 1731521749017 2024-11-13T18:15:49,018 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 644025ac5ad44a7ca4a26c89d6611d8b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741855_1031 (size=110) 2024-11-13T18:15:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741855_1031 (size=110) 2024-11-13T18:15:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741855_1031 (size=110) 2024-11-13T18:15:49,044 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,044 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 64b5086eb7020ccba10769de2eaf3d9b, disabling compactions & flushes 2024-11-13T18:15:49,044 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:49,045 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:49,045 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. after waiting 0 ms 2024-11-13T18:15:49,045 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:49,045 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:49,045 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 64b5086eb7020ccba10769de2eaf3d9b: Waiting for close lock at 1731521749044Disabling compacts and flushes for region at 1731521749044Disabling writes for close at 1731521749045 (+1 ms)Writing region close event to WAL at 1731521749045Closed at 1731521749045 2024-11-13T18:15:49,046 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fb720a25eb86c541f9c1f814843c651b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741856_1032 (size=110) 2024-11-13T18:15:49,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741856_1032 (size=110) 2024-11-13T18:15:49,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741856_1032 (size=110) 2024-11-13T18:15:49,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741857_1033 (size=110) 2024-11-13T18:15:49,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741857_1033 (size=110) 2024-11-13T18:15:49,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741857_1033 (size=110) 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 1b91248305f9bd170badfc1ac16c2f2b, disabling compactions & flushes 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 644025ac5ad44a7ca4a26c89d6611d8b, disabling compactions & flushes 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. after waiting 0 ms 2024-11-13T18:15:49,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. after waiting 0 ms 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1b91248305f9bd170badfc1ac16c2f2b: Waiting for close lock at 1731521749071Disabling compacts and flushes for region at 1731521749071Disabling writes for close at 1731521749071Writing region close event to WAL at 1731521749071Closed at 1731521749071 2024-11-13T18:15:49,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 644025ac5ad44a7ca4a26c89d6611d8b: Waiting for close lock at 1731521749071Disabling compacts and flushes for region at 1731521749071Disabling writes for close at 1731521749071Writing region close event to WAL at 1731521749071Closed at 1731521749071 2024-11-13T18:15:49,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5a59239e975bc7ad59b0ae9308416d51, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,073 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b196a2f44f4f468f8bb9a2e59b301747, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741858_1034 (size=110) 2024-11-13T18:15:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741858_1034 (size=110) 2024-11-13T18:15:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741858_1034 (size=110) 2024-11-13T18:15:49,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing fb720a25eb86c541f9c1f814843c651b, disabling compactions & flushes 2024-11-13T18:15:49,079 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. after waiting 0 ms 2024-11-13T18:15:49,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,080 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for fb720a25eb86c541f9c1f814843c651b: Waiting for close lock at 1731521749079Disabling compacts and flushes for region at 1731521749079Disabling writes for close at 1731521749080 (+1 ms)Writing region close event to WAL at 1731521749080Closed at 1731521749080 2024-11-13T18:15:49,081 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 400b285d3ec5b0d43bcd37cdfa6b8f90, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741859_1035 (size=110) 2024-11-13T18:15:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741859_1035 (size=110) 2024-11-13T18:15:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741859_1035 (size=110) 2024-11-13T18:15:49,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing b196a2f44f4f468f8bb9a2e59b301747, disabling compactions & flushes 2024-11-13T18:15:49,093 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. after waiting 0 ms 2024-11-13T18:15:49,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,093 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,094 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for b196a2f44f4f468f8bb9a2e59b301747: Waiting for close lock at 1731521749093Disabling compacts and flushes for region at 1731521749093Disabling writes for close at 1731521749093Writing region close event to WAL at 1731521749093Closed at 1731521749093 2024-11-13T18:15:49,095 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 272983331631897f6cfeac3ce6bd7466, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741860_1036 (size=110) 2024-11-13T18:15:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741860_1036 (size=110) 2024-11-13T18:15:49,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741860_1036 (size=110) 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 5a59239e975bc7ad59b0ae9308416d51, disabling compactions & flushes 2024-11-13T18:15:49,102 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. after waiting 0 ms 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,102 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5a59239e975bc7ad59b0ae9308416d51: Waiting for close lock at 1731521749102Disabling compacts and flushes for region at 1731521749102Disabling writes for close at 1731521749102Writing region close event to WAL at 1731521749102Closed at 1731521749102 2024-11-13T18:15:49,103 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cf5aa9737456e8265a9b1e93d818d677, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada 2024-11-13T18:15:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741861_1037 (size=110) 2024-11-13T18:15:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741861_1037 (size=110) 2024-11-13T18:15:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741861_1037 (size=110) 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 400b285d3ec5b0d43bcd37cdfa6b8f90, disabling compactions & flushes 2024-11-13T18:15:49,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. after waiting 0 ms 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:49,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:49,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 400b285d3ec5b0d43bcd37cdfa6b8f90: Waiting for close lock at 1731521749115Disabling compacts and flushes for region at 1731521749115Disabling writes for close at 1731521749115Writing region close event to WAL at 1731521749115Closed at 1731521749115 2024-11-13T18:15:49,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741862_1038 (size=110) 2024-11-13T18:15:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741862_1038 (size=110) 2024-11-13T18:15:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741862_1038 (size=110) 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 272983331631897f6cfeac3ce6bd7466, disabling compactions & flushes 2024-11-13T18:15:49,122 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. after waiting 0 ms 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:49,122 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:49,122 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 272983331631897f6cfeac3ce6bd7466: Waiting for close lock at 1731521749122Disabling compacts and flushes for region at 1731521749122Disabling writes for close at 1731521749122Writing region close event to WAL at 1731521749122Closed at 1731521749122 2024-11-13T18:15:49,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741863_1039 (size=107) 2024-11-13T18:15:49,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741863_1039 (size=107) 2024-11-13T18:15:49,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741863_1039 (size=107) 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing cf5aa9737456e8265a9b1e93d818d677, disabling compactions & flushes 2024-11-13T18:15:49,130 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. after waiting 0 ms 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,130 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,130 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for cf5aa9737456e8265a9b1e93d818d677: Waiting for close lock at 1731521749130Disabling compacts and flushes for region at 1731521749130Disabling writes for close at 1731521749130Writing region close event to WAL at 1731521749130Closed at 1731521749130 2024-11-13T18:15:49,133 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:15:49,141 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,141 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,141 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731521749134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731521749134"}]},"ts":"1731521749134"} 2024-11-13T18:15:49,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-13T18:15:49,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:15:49,202 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521749200"}]},"ts":"1731521749200"} 2024-11-13T18:15:49,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-13T18:15:49,208 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:15:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T18:15:49,275 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:15:49,275 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:15:49,275 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T18:15:49,275 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:15:49,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd437c88e4af73574ce0ce0882aca142, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=38c892b7353daa24a9b6c917f1aa0a89, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00a4d576f4c41497f18224865b73403a, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1e715c3850b0001604b62543d4e17908, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=801d0f207e45efea50135a16f5fcbc99, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5c39d6007aedf97e4c4a94511fe061ef, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=27ae2919589ed9f92a8ad04abb8e24d5, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=575404eded2e7a15cbfa9186eea1d1b1, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dae4cf552a1afaf0b041668499d99e20, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1b91248305f9bd170badfc1ac16c2f2b, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=644025ac5ad44a7ca4a26c89d6611d8b, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5a59239e975bc7ad59b0ae9308416d51, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b196a2f44f4f468f8bb9a2e59b301747, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677, ASSIGN}] 2024-11-13T18:15:49,290 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677, ASSIGN 2024-11-13T18:15:49,291 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,293 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466, ASSIGN 2024-11-13T18:15:49,295 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,297 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN 2024-11-13T18:15:49,299 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,301 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b196a2f44f4f468f8bb9a2e59b301747, ASSIGN 2024-11-13T18:15:49,303 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b196a2f44f4f468f8bb9a2e59b301747, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,304 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5a59239e975bc7ad59b0ae9308416d51, ASSIGN 2024-11-13T18:15:49,306 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5a59239e975bc7ad59b0ae9308416d51, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,307 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN 2024-11-13T18:15:49,309 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,310 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=644025ac5ad44a7ca4a26c89d6611d8b, ASSIGN 2024-11-13T18:15:49,312 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=644025ac5ad44a7ca4a26c89d6611d8b, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,313 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1b91248305f9bd170badfc1ac16c2f2b, ASSIGN 2024-11-13T18:15:49,314 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1b91248305f9bd170badfc1ac16c2f2b, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN 2024-11-13T18:15:49,319 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dae4cf552a1afaf0b041668499d99e20, ASSIGN 2024-11-13T18:15:49,323 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dae4cf552a1afaf0b041668499d99e20, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,324 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=575404eded2e7a15cbfa9186eea1d1b1, ASSIGN 2024-11-13T18:15:49,326 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=575404eded2e7a15cbfa9186eea1d1b1, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,328 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN 2024-11-13T18:15:49,329 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,331 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796, ASSIGN 2024-11-13T18:15:49,332 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,335 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=27ae2919589ed9f92a8ad04abb8e24d5, ASSIGN 2024-11-13T18:15:49,337 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=27ae2919589ed9f92a8ad04abb8e24d5, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,339 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN 2024-11-13T18:15:49,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1, ASSIGN 2024-11-13T18:15:49,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e, ASSIGN 2024-11-13T18:15:49,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,349 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN 2024-11-13T18:15:49,351 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,352 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5c39d6007aedf97e4c4a94511fe061ef, ASSIGN 2024-11-13T18:15:49,355 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5c39d6007aedf97e4c4a94511fe061ef, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,357 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=801d0f207e45efea50135a16f5fcbc99, ASSIGN 2024-11-13T18:15:49,359 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=801d0f207e45efea50135a16f5fcbc99, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN 2024-11-13T18:15:49,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1e715c3850b0001604b62543d4e17908, ASSIGN 2024-11-13T18:15:49,365 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1e715c3850b0001604b62543d4e17908, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,367 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00a4d576f4c41497f18224865b73403a, ASSIGN 2024-11-13T18:15:49,369 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00a4d576f4c41497f18224865b73403a, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,370 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN 2024-11-13T18:15:49,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN; state=OFFLINE, location=fda96cf025dc,42475,1731521745166; forceNewPlan=false, retain=false 2024-11-13T18:15:49,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=38c892b7353daa24a9b6c917f1aa0a89, ASSIGN 2024-11-13T18:15:49,376 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=38c892b7353daa24a9b6c917f1aa0a89, ASSIGN; state=OFFLINE, location=fda96cf025dc,35435,1731521744991; forceNewPlan=false, retain=false 2024-11-13T18:15:49,382 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd437c88e4af73574ce0ce0882aca142, ASSIGN 2024-11-13T18:15:49,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd437c88e4af73574ce0ce0882aca142, ASSIGN; state=OFFLINE, location=fda96cf025dc,46791,1731521745104; forceNewPlan=false, retain=false 2024-11-13T18:15:49,446 INFO [fda96cf025dc:41233 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-13T18:15:49,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=cf5aa9737456e8265a9b1e93d818d677, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,449 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5a59239e975bc7ad59b0ae9308416d51, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=37863ce38db650b86b62a2bccc5e7796, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677, ASSIGN because future has completed 2024-11-13T18:15:49,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5aa9737456e8265a9b1e93d818d677, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5a59239e975bc7ad59b0ae9308416d51, ASSIGN because future has completed 2024-11-13T18:15:49,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=644025ac5ad44a7ca4a26c89d6611d8b, ASSIGN because future has completed 2024-11-13T18:15:49,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=575404eded2e7a15cbfa9186eea1d1b1, ASSIGN because future has completed 2024-11-13T18:15:49,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 644025ac5ad44a7ca4a26c89d6611d8b, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 575404eded2e7a15cbfa9186eea1d1b1, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796, ASSIGN because future has completed 2024-11-13T18:15:49,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e, ASSIGN because future has completed 2024-11-13T18:15:49,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5aa68e8ac635bc9e7d6868f5f771281e, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37863ce38db650b86b62a2bccc5e7796, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a59239e975bc7ad59b0ae9308416d51, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1e715c3850b0001604b62543d4e17908, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd437c88e4af73574ce0ce0882aca142, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,507 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fb720a25eb86c541f9c1f814843c651b, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,508 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,508 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,509 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8636f69311e6a46aa80948ea8be00961, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5c39d6007aedf97e4c4a94511fe061ef, ASSIGN because future has completed 2024-11-13T18:15:49,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=09ff4ab7a9d0091782b93c194b270c43, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=02893942b5e22ded04197e4e8486e0dc, regionState=OPENING, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=272983331631897f6cfeac3ce6bd7466, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,515 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=dae4cf552a1afaf0b041668499d99e20, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=801d0f207e45efea50135a16f5fcbc99, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=00a4d576f4c41497f18224865b73403a, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1e715c3850b0001604b62543d4e17908, ASSIGN because future has completed 2024-11-13T18:15:49,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e715c3850b0001604b62543d4e17908, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd437c88e4af73574ce0ce0882aca142, ASSIGN because future has completed 2024-11-13T18:15:49,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN because future has completed 2024-11-13T18:15:49,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN because future has completed 2024-11-13T18:15:49,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN because future has completed 2024-11-13T18:15:49,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN because future has completed 2024-11-13T18:15:49,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN because future has completed 2024-11-13T18:15:49,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN because future has completed 2024-11-13T18:15:49,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN because future has completed 2024-11-13T18:15:49,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN because future has completed 2024-11-13T18:15:49,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466, ASSIGN because future has completed 2024-11-13T18:15:49,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272983331631897f6cfeac3ce6bd7466, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b196a2f44f4f468f8bb9a2e59b301747, ASSIGN because future has completed 2024-11-13T18:15:49,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1b91248305f9bd170badfc1ac16c2f2b, ASSIGN because future has completed 2024-11-13T18:15:49,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dae4cf552a1afaf0b041668499d99e20, ASSIGN because future has completed 2024-11-13T18:15:49,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b91248305f9bd170badfc1ac16c2f2b, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=27ae2919589ed9f92a8ad04abb8e24d5, ASSIGN because future has completed 2024-11-13T18:15:49,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1, ASSIGN because future has completed 2024-11-13T18:15:49,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=801d0f207e45efea50135a16f5fcbc99, ASSIGN because future has completed 2024-11-13T18:15:49,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00a4d576f4c41497f18224865b73403a, ASSIGN because future has completed 2024-11-13T18:15:49,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a4d576f4c41497f18224865b73403a, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 801d0f207e45efea50135a16f5fcbc99, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5bab85a9454bcf58ade3574335c3bd1, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ae2919589ed9f92a8ad04abb8e24d5, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,607 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:15:49,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure dae4cf552a1afaf0b041668499d99e20, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43657, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:15:49,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure b196a2f44f4f468f8bb9a2e59b301747, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,632 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,632 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a59239e975bc7ad59b0ae9308416d51, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-13T18:15:49,633 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,633 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,633 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,633 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,643 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,646 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a59239e975bc7ad59b0ae9308416d51 columnFamilyName colfamily11 2024-11-13T18:15:49,646 DEBUG [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,649 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(327): Store=5a59239e975bc7ad59b0ae9308416d51/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,649 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,652 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a59239e975bc7ad59b0ae9308416d51 columnFamilyName colfamily21 2024-11-13T18:15:49,652 DEBUG [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,653 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(327): Store=5a59239e975bc7ad59b0ae9308416d51/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,653 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,657 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a59239e975bc7ad59b0ae9308416d51 columnFamilyName colfamily31 2024-11-13T18:15:49,657 DEBUG [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,658 INFO [StoreOpener-5a59239e975bc7ad59b0ae9308416d51-1 {}] regionserver.HStore(327): Store=5a59239e975bc7ad59b0ae9308416d51/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,663 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,669 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,670 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,42475,1731521745166}] 2024-11-13T18:15:49,674 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,674 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,675 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,678 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd437c88e4af73574ce0ce0882aca142, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c39d6007aedf97e4c4a94511fe061ef, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:15:49,694 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=38c892b7353daa24a9b6c917f1aa0a89, ASSIGN because future has completed 2024-11-13T18:15:49,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38c892b7353daa24a9b6c917f1aa0a89, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:15:49,706 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,707 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 5a59239e975bc7ad59b0ae9308416d51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64624817, jitterRate=-0.03701518476009369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,707 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:15:49,708 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 5a59239e975bc7ad59b0ae9308416d51: Running coprocessor pre-open hook at 1731521749633Writing region info on filesystem at 1731521749633Initializing all the Stores at 1731521749639 (+6 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749639Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749642 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749642Cleaning up temporary data from old regions at 1731521749674 (+32 ms)Running coprocessor post-open hooks at 1731521749707 (+33 ms)Region opened successfully at 1731521749708 (+1 ms) 2024-11-13T18:15:49,710 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., pid=36, masterSystemTime=1731521749607 2024-11-13T18:15:49,714 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:15:49,714 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,714 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:15:49,714 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:49,714 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e715c3850b0001604b62543d4e17908, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-13T18:15:49,715 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,715 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,716 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,716 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a59239e975bc7ad59b0ae9308416d51, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:49,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:15:49,722 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:49,722 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,722 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eefcb355d011c4ea65b0572629d5ef5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T18:15:49,723 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,723 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,723 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,723 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,728 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e715c3850b0001604b62543d4e17908 columnFamilyName colfamily11 2024-11-13T18:15:49,728 DEBUG [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,729 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(327): Store=1e715c3850b0001604b62543d4e17908/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,729 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,731 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e715c3850b0001604b62543d4e17908 columnFamilyName colfamily21 2024-11-13T18:15:49,732 DEBUG [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,732 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(327): Store=1e715c3850b0001604b62543d4e17908/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,733 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,735 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e715c3850b0001604b62543d4e17908 columnFamilyName colfamily31 2024-11-13T18:15:49,735 DEBUG [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:49,738 INFO [StoreOpener-1e715c3850b0001604b62543d4e17908-1 {}] regionserver.HStore(327): Store=1e715c3850b0001604b62543d4e17908/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,738 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,740 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,741 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => b196a2f44f4f468f8bb9a2e59b301747, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-13T18:15:49,741 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,741 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,741 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,742 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,742 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,742 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=26 2024-11-13T18:15:49,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a59239e975bc7ad59b0ae9308416d51, server=fda96cf025dc,46791,1731521745104 in 223 msec 2024-11-13T18:15:49,744 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,744 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,745 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5a59239e975bc7ad59b0ae9308416d51, ASSIGN in 465 msec 2024-11-13T18:15:49,747 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,748 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily11 2024-11-13T18:15:49,748 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,749 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,750 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,750 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,754 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily21 2024-11-13T18:15:49,754 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,755 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,755 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,757 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily31 2024-11-13T18:15:49,757 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,758 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,758 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,759 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,760 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,761 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,763 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,763 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,763 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,766 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,770 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,771 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 3eefcb355d011c4ea65b0572629d5ef5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65066108, jitterRate=-0.030439436435699463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,771 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:15:49,771 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 3eefcb355d011c4ea65b0572629d5ef5: Running coprocessor pre-open hook at 1731521749723Writing region info on filesystem at 1731521749723Initializing all the Stores at 1731521749737 (+14 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749737Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749743 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749743Cleaning up temporary data from old regions at 1731521749763 (+20 ms)Running coprocessor post-open hooks at 1731521749771 (+8 ms)Region opened successfully at 1731521749771 2024-11-13T18:15:49,772 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,773 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 1e715c3850b0001604b62543d4e17908; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59503721, jitterRate=-0.11332546174526215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,773 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:15:49,774 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 1e715c3850b0001604b62543d4e17908: Running coprocessor pre-open hook at 1731521749716Writing region info on filesystem at 1731521749716Initializing all the Stores at 1731521749717 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749718 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749722 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749722Cleaning up temporary data from old regions at 1731521749745 (+23 ms)Running coprocessor post-open hooks at 1731521749774 (+29 ms)Region opened successfully at 1731521749774 2024-11-13T18:15:49,781 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., pid=50, masterSystemTime=1731521749708 2024-11-13T18:15:49,781 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., pid=37, masterSystemTime=1731521749607 2024-11-13T18:15:49,786 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:49,786 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:15:49,786 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:49,786 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b196a2f44f4f468f8bb9a2e59b301747 columnFamilyName colfamily11 2024-11-13T18:15:49,787 DEBUG [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,787 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 09ff4ab7a9d0091782b93c194b270c43, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T18:15:49,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,787 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,787 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,787 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,787 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,788 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(327): Store=b196a2f44f4f468f8bb9a2e59b301747/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,789 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,789 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:49,789 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:15:49,789 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,789 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 644025ac5ad44a7ca4a26c89d6611d8b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-13T18:15:49,790 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,790 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,790 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,790 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,792 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1e715c3850b0001604b62543d4e17908, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,799 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b196a2f44f4f468f8bb9a2e59b301747 columnFamilyName colfamily21 2024-11-13T18:15:49,799 DEBUG [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:49,800 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(327): Store=b196a2f44f4f468f8bb9a2e59b301747/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,800 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e715c3850b0001604b62543d4e17908, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:49,803 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b196a2f44f4f468f8bb9a2e59b301747 columnFamilyName colfamily31 2024-11-13T18:15:49,803 DEBUG [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=19 2024-11-13T18:15:49,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,42475,1731521745166 in 153 msec 2024-11-13T18:15:49,809 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN in 529 msec 2024-11-13T18:15:49,812 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 644025ac5ad44a7ca4a26c89d6611d8b columnFamilyName colfamily11 2024-11-13T18:15:49,812 DEBUG [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,812 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(327): Store=644025ac5ad44a7ca4a26c89d6611d8b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,813 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,815 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 644025ac5ad44a7ca4a26c89d6611d8b columnFamilyName colfamily21 2024-11-13T18:15:49,815 DEBUG [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=9 2024-11-13T18:15:49,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e715c3850b0001604b62543d4e17908, server=fda96cf025dc,46791,1731521745104 in 290 msec 2024-11-13T18:15:49,816 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(327): Store=644025ac5ad44a7ca4a26c89d6611d8b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,816 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,818 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 644025ac5ad44a7ca4a26c89d6611d8b columnFamilyName colfamily31 2024-11-13T18:15:49,818 DEBUG [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1e715c3850b0001604b62543d4e17908, ASSIGN in 538 msec 2024-11-13T18:15:49,819 INFO [StoreOpener-644025ac5ad44a7ca4a26c89d6611d8b-1 {}] regionserver.HStore(327): Store=644025ac5ad44a7ca4a26c89d6611d8b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,819 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,820 INFO [StoreOpener-b196a2f44f4f468f8bb9a2e59b301747-1 {}] regionserver.HStore(327): Store=b196a2f44f4f468f8bb9a2e59b301747/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,820 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,821 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,821 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,821 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,823 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily11 2024-11-13T18:15:49,823 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,824 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,824 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,824 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,824 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,825 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,825 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,826 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,829 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,829 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,830 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily21 2024-11-13T18:15:49,830 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,831 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,831 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,846 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,846 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,848 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,853 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily31 2024-11-13T18:15:49,853 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,854 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,854 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,855 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,856 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,858 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,858 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,859 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,861 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,876 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,878 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 644025ac5ad44a7ca4a26c89d6611d8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72541743, jitterRate=0.08095620572566986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,878 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:15:49,878 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 644025ac5ad44a7ca4a26c89d6611d8b: Running coprocessor pre-open hook at 1731521749790Writing region info on filesystem at 1731521749790Initializing all the Stores at 1731521749799 (+9 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749799Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749808 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749808Cleaning up temporary data from old regions at 1731521749824 (+16 ms)Running coprocessor post-open hooks at 1731521749878 (+54 ms)Region opened successfully at 1731521749878 2024-11-13T18:15:49,881 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., pid=32, masterSystemTime=1731521749607 2024-11-13T18:15:49,882 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,884 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened b196a2f44f4f468f8bb9a2e59b301747; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63199545, jitterRate=-0.058253392577171326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,884 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:15:49,884 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for b196a2f44f4f468f8bb9a2e59b301747: Running coprocessor pre-open hook at 1731521749742Writing region info on filesystem at 1731521749742Initializing all the Stores at 1731521749744 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749744Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749760 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749760Cleaning up temporary data from old regions at 1731521749830 (+70 ms)Running coprocessor post-open hooks at 1731521749884 (+54 ms)Region opened successfully at 1731521749884 2024-11-13T18:15:49,886 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,886 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:15:49,886 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,887 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => cf5aa9737456e8265a9b1e93d818d677, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-13T18:15:49,887 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,887 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,887 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., pid=47, masterSystemTime=1731521749713 2024-11-13T18:15:49,888 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,888 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,893 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 644025ac5ad44a7ca4a26c89d6611d8b, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:49,894 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 09ff4ab7a9d0091782b93c194b270c43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70066832, jitterRate=0.044077157974243164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,894 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:15:49,894 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 09ff4ab7a9d0091782b93c194b270c43: Running coprocessor pre-open hook at 1731521749788Writing region info on filesystem at 1731521749788Initializing all the Stores at 1731521749801 (+13 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749801Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749821 (+20 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749821Cleaning up temporary data from old regions at 1731521749858 (+37 ms)Running coprocessor post-open hooks at 1731521749894 (+36 ms)Region opened successfully at 1731521749894 2024-11-13T18:15:49,895 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,895 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:15:49,895 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:49,896 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => e5bab85a9454bcf58ade3574335c3bd1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-13T18:15:49,896 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,896 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,897 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,897 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,897 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., pid=39, masterSystemTime=1731521749708 2024-11-13T18:15:49,899 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,900 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=24 2024-11-13T18:15:49,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 644025ac5ad44a7ca4a26c89d6611d8b, server=fda96cf025dc,46791,1731521745104 in 438 msec 2024-11-13T18:15:49,904 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:49,904 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:15:49,904 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,905 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => fb720a25eb86c541f9c1f814843c651b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T18:15:49,905 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5aa9737456e8265a9b1e93d818d677 columnFamilyName colfamily11 2024-11-13T18:15:49,905 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,905 DEBUG [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,905 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,906 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,906 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,907 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5bab85a9454bcf58ade3574335c3bd1 columnFamilyName colfamily11 2024-11-13T18:15:49,907 DEBUG [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,908 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,911 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(327): Store=e5bab85a9454bcf58ade3574335c3bd1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,911 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(327): Store=cf5aa9737456e8265a9b1e93d818d677/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,911 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:49,913 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,913 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily11 2024-11-13T18:15:49,913 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,914 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,914 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5aa9737456e8265a9b1e93d818d677 columnFamilyName colfamily21 2024-11-13T18:15:49,914 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,914 DEBUG [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,915 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(327): Store=cf5aa9737456e8265a9b1e93d818d677/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,915 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,915 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5bab85a9454bcf58ade3574335c3bd1 columnFamilyName colfamily21 2024-11-13T18:15:49,915 DEBUG [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,916 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(327): Store=e5bab85a9454bcf58ade3574335c3bd1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,917 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily21 2024-11-13T18:15:49,917 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,917 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,917 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5aa9737456e8265a9b1e93d818d677 columnFamilyName colfamily31 2024-11-13T18:15:49,917 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,918 DEBUG [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,918 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=10 2024-11-13T18:15:49,919 INFO [StoreOpener-cf5aa9737456e8265a9b1e93d818d677-1 {}] regionserver.HStore(327): Store=cf5aa9737456e8265a9b1e93d818d677/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,42475,1731521745166 in 366 msec 2024-11-13T18:15:49,919 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,919 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5bab85a9454bcf58ade3574335c3bd1 columnFamilyName colfamily31 2024-11-13T18:15:49,919 DEBUG [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,920 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily31 2024-11-13T18:15:49,920 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,920 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,921 INFO [StoreOpener-e5bab85a9454bcf58ade3574335c3bd1-1 {}] regionserver.HStore(327): Store=e5bab85a9454bcf58ade3574335c3bd1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,921 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,922 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,923 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN in 641 msec 2024-11-13T18:15:49,923 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,924 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,924 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,925 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,926 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,926 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=644025ac5ad44a7ca4a26c89d6611d8b, ASSIGN in 645 msec 2024-11-13T18:15:49,927 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,930 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,931 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,932 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=47, ppid=27, state=RUNNABLE, hasLock=true; OpenRegionProcedure b196a2f44f4f468f8bb9a2e59b301747, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,933 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,933 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,934 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,935 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=27 2024-11-13T18:15:49,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure b196a2f44f4f468f8bb9a2e59b301747, server=fda96cf025dc,35435,1731521744991 in 312 msec 2024-11-13T18:15:49,937 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,937 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,937 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,938 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,938 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened cf5aa9737456e8265a9b1e93d818d677; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72576304, jitterRate=0.08147120475769043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,939 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:15:49,939 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for cf5aa9737456e8265a9b1e93d818d677: Running coprocessor pre-open hook at 1731521749888Writing region info on filesystem at 1731521749888Initializing all the Stores at 1731521749889 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749890 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749898 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749898Cleaning up temporary data from old regions at 1731521749924 (+26 ms)Running coprocessor post-open hooks at 1731521749939 (+15 ms)Region opened successfully at 1731521749939 2024-11-13T18:15:49,940 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,940 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677., pid=31, masterSystemTime=1731521749607 2024-11-13T18:15:49,940 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,941 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened e5bab85a9454bcf58ade3574335c3bd1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64235194, jitterRate=-0.042821019887924194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,941 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:15:49,941 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for e5bab85a9454bcf58ade3574335c3bd1: Running coprocessor pre-open hook at 1731521749897Writing region info on filesystem at 1731521749897Initializing all the Stores at 1731521749899 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749899Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749900 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749900Cleaning up temporary data from old regions at 1731521749927 (+27 ms)Running coprocessor post-open hooks at 1731521749941 (+14 ms)Region opened successfully at 1731521749941 2024-11-13T18:15:49,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b196a2f44f4f468f8bb9a2e59b301747, ASSIGN in 659 msec 2024-11-13T18:15:49,942 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., pid=44, masterSystemTime=1731521749713 2024-11-13T18:15:49,944 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,944 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:15:49,945 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,945 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:49,945 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 575404eded2e7a15cbfa9186eea1d1b1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,946 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,946 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:49,946 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b91248305f9bd170badfc1ac16c2f2b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T18:15:49,947 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,947 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,947 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,947 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:49,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,46791,1731521745104, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:49,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5aa9737456e8265a9b1e93d818d677, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:49,953 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5bab85a9454bcf58ade3574335c3bd1, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:49,954 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened fb720a25eb86c541f9c1f814843c651b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67166564, jitterRate=8.597970008850098E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,954 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:15:49,954 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for fb720a25eb86c541f9c1f814843c651b: Running coprocessor pre-open hook at 1731521749906Writing region info on filesystem at 1731521749906Initializing all the Stores at 1731521749907 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749907Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749908 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749908Cleaning up temporary data from old regions at 1731521749937 (+29 ms)Running coprocessor post-open hooks at 1731521749954 (+17 ms)Region opened successfully at 1731521749954 2024-11-13T18:15:49,956 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,35435,1731521744991, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:49,956 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., pid=52, masterSystemTime=1731521749708 2024-11-13T18:15:49,957 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,958 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,959 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 575404eded2e7a15cbfa9186eea1d1b1 columnFamilyName colfamily11 2024-11-13T18:15:49,960 DEBUG [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,960 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b91248305f9bd170badfc1ac16c2f2b columnFamilyName colfamily11 2024-11-13T18:15:49,960 DEBUG [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,961 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,961 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:15:49,961 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(327): Store=1b91248305f9bd170badfc1ac16c2f2b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,961 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:49,962 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,962 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(327): Store=575404eded2e7a15cbfa9186eea1d1b1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fb2e398d735ceeb281622c9b30d4cf1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T18:15:49,962 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-13T18:15:49,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure cf5aa9737456e8265a9b1e93d818d677, server=fda96cf025dc,46791,1731521745104 in 502 msec 2024-11-13T18:15:49,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,965 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 575404eded2e7a15cbfa9186eea1d1b1 columnFamilyName colfamily21 2024-11-13T18:15:49,965 DEBUG [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,966 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,966 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(327): Store=575404eded2e7a15cbfa9186eea1d1b1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf5aa9737456e8265a9b1e93d818d677, ASSIGN in 684 msec 2024-11-13T18:15:49,967 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b91248305f9bd170badfc1ac16c2f2b columnFamilyName colfamily21 2024-11-13T18:15:49,967 DEBUG [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:49,969 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily11 2024-11-13T18:15:49,969 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,969 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,969 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(327): Store=1b91248305f9bd170badfc1ac16c2f2b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,969 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,971 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:49,972 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,972 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b91248305f9bd170badfc1ac16c2f2b columnFamilyName colfamily31 2024-11-13T18:15:49,972 DEBUG [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,973 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 575404eded2e7a15cbfa9186eea1d1b1 columnFamilyName colfamily31 2024-11-13T18:15:49,973 DEBUG [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,974 INFO [StoreOpener-1b91248305f9bd170badfc1ac16c2f2b-1 {}] regionserver.HStore(327): Store=1b91248305f9bd170badfc1ac16c2f2b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,974 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,974 INFO [StoreOpener-575404eded2e7a15cbfa9186eea1d1b1-1 {}] regionserver.HStore(327): Store=575404eded2e7a15cbfa9186eea1d1b1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,974 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,975 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily21 2024-11-13T18:15:49,975 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,975 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,975 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,975 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,976 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,976 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,976 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,978 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,978 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,978 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily31 2024-11-13T18:15:49,978 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:49,978 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,978 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,979 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,979 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:49,979 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,979 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,981 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,982 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,982 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,982 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=15 2024-11-13T18:15:49,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure e5bab85a9454bcf58ade3574335c3bd1, server=fda96cf025dc,35435,1731521744991 in 372 msec 2024-11-13T18:15:49,986 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,986 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e5bab85a9454bcf58ade3574335c3bd1, ASSIGN in 706 msec 2024-11-13T18:15:49,987 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:49,989 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,990 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:49,990 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 575404eded2e7a15cbfa9186eea1d1b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64140558, jitterRate=-0.04423120617866516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,990 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:15:49,991 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 575404eded2e7a15cbfa9186eea1d1b1: Running coprocessor pre-open hook at 1731521749946Writing region info on filesystem at 1731521749946Initializing all the Stores at 1731521749951 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749952 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749957 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749957Cleaning up temporary data from old regions at 1731521749978 (+21 ms)Running coprocessor post-open hooks at 1731521749990 (+12 ms)Region opened successfully at 1731521749990 2024-11-13T18:15:49,992 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., pid=33, masterSystemTime=1731521749607 2024-11-13T18:15:49,995 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,995 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:15:49,995 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:49,995 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 37863ce38db650b86b62a2bccc5e7796, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-13T18:15:49,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=25 2024-11-13T18:15:49,996 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:49,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,42475,1731521745166 in 326 msec 2024-11-13T18:15:49,996 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:49,996 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:49,996 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:49,997 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:49,998 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 1b91248305f9bd170badfc1ac16c2f2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63216088, jitterRate=-0.058006882667541504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:49,998 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:15:49,998 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 1b91248305f9bd170badfc1ac16c2f2b: Running coprocessor pre-open hook at 1731521749950Writing region info on filesystem at 1731521749950Initializing all the Stores at 1731521749951 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749952 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749957 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749957Cleaning up temporary data from old regions at 1731521749978 (+21 ms)Running coprocessor post-open hooks at 1731521749998 (+20 ms)Region opened successfully at 1731521749998 2024-11-13T18:15:49,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN in 718 msec 2024-11-13T18:15:50,000 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., pid=41, masterSystemTime=1731521749713 2024-11-13T18:15:50,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:50,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 575404eded2e7a15cbfa9186eea1d1b1, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:50,004 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:50,004 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:15:50,005 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:50,005 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 27ae2919589ed9f92a8ad04abb8e24d5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T18:15:50,006 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,006 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,006 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,006 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,006 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,008 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37863ce38db650b86b62a2bccc5e7796 columnFamilyName colfamily11 2024-11-13T18:15:50,008 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,008 DEBUG [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,008 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,011 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae2919589ed9f92a8ad04abb8e24d5 columnFamilyName colfamily11 2024-11-13T18:15:50,011 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 7fb2e398d735ceeb281622c9b30d4cf1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72958798, jitterRate=0.08717080950737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,011 DEBUG [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,011 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:15:50,011 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(327): Store=37863ce38db650b86b62a2bccc5e7796/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,011 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 7fb2e398d735ceeb281622c9b30d4cf1: Running coprocessor pre-open hook at 1731521749963Writing region info on filesystem at 1731521749963Initializing all the Stores at 1731521749965 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749965Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749966 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749966Cleaning up temporary data from old regions at 1731521749986 (+20 ms)Running coprocessor post-open hooks at 1731521750011 (+25 ms)Region opened successfully at 1731521750011 2024-11-13T18:15:50,011 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,012 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(327): Store=27ae2919589ed9f92a8ad04abb8e24d5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,012 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b91248305f9bd170badfc1ac16c2f2b, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,013 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., pid=49, masterSystemTime=1731521749708 2024-11-13T18:15:50,014 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae2919589ed9f92a8ad04abb8e24d5 columnFamilyName colfamily21 2024-11-13T18:15:50,014 DEBUG [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,014 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(327): Store=27ae2919589ed9f92a8ad04abb8e24d5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,015 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,019 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae2919589ed9f92a8ad04abb8e24d5 columnFamilyName colfamily31 2024-11-13T18:15:50,019 DEBUG [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:50,020 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:15:50,020 INFO [StoreOpener-27ae2919589ed9f92a8ad04abb8e24d5-1 {}] regionserver.HStore(327): Store=27ae2919589ed9f92a8ad04abb8e24d5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=20 2024-11-13T18:15:50,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 575404eded2e7a15cbfa9186eea1d1b1, server=fda96cf025dc,46791,1731521745104 in 548 msec 2024-11-13T18:15:50,021 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:50,021 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 400b285d3ec5b0d43bcd37cdfa6b8f90, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-13T18:15:50,021 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37863ce38db650b86b62a2bccc5e7796 columnFamilyName colfamily21 2024-11-13T18:15:50,021 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,021 DEBUG [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,021 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,022 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,022 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,022 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(327): Store=37863ce38db650b86b62a2bccc5e7796/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,022 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,023 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,024 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=575404eded2e7a15cbfa9186eea1d1b1, ASSIGN in 743 msec 2024-11-13T18:15:50,025 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,025 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,026 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,026 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37863ce38db650b86b62a2bccc5e7796 columnFamilyName colfamily31 2024-11-13T18:15:50,027 DEBUG [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,027 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,027 INFO [StoreOpener-37863ce38db650b86b62a2bccc5e7796-1 {}] regionserver.HStore(327): Store=37863ce38db650b86b62a2bccc5e7796/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,028 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:50,029 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,030 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,032 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,032 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:50,033 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,034 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily11 2024-11-13T18:15:50,035 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,035 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,037 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,037 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-11-13T18:15:50,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b91248305f9bd170badfc1ac16c2f2b, server=fda96cf025dc,35435,1731521744991 in 467 msec 2024-11-13T18:15:50,040 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily21 2024-11-13T18:15:50,040 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1b91248305f9bd170badfc1ac16c2f2b, ASSIGN in 759 msec 2024-11-13T18:15:50,040 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,041 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,043 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily31 2024-11-13T18:15:50,043 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,043 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,044 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=13 2024-11-13T18:15:50,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,42475,1731521745166 in 405 msec 2024-11-13T18:15:50,045 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,045 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,047 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN in 767 msec 2024-11-13T18:15:50,047 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,048 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,050 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,050 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,053 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 27ae2919589ed9f92a8ad04abb8e24d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65468375, jitterRate=-0.024445191025733948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,053 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:15:50,054 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 27ae2919589ed9f92a8ad04abb8e24d5: Running coprocessor pre-open hook at 1731521750006Writing region info on filesystem at 1731521750006Initializing all the Stores at 1731521750008 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750008Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750008Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750008Cleaning up temporary data from old regions at 1731521750025 (+17 ms)Running coprocessor post-open hooks at 1731521750053 (+28 ms)Region opened successfully at 1731521750054 (+1 ms) 2024-11-13T18:15:50,055 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., pid=45, masterSystemTime=1731521749713 2024-11-13T18:15:50,061 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:50,061 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:15:50,061 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:50,061 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 272983331631897f6cfeac3ce6bd7466, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T18:15:50,062 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,062 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,062 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,062 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,062 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,063 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,063 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 37863ce38db650b86b62a2bccc5e7796; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65821757, jitterRate=-0.019179388880729675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,063 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:15:50,063 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 37863ce38db650b86b62a2bccc5e7796: Running coprocessor pre-open hook at 1731521749996Writing region info on filesystem at 1731521749996Initializing all the Stores at 1731521749998 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521749998Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750005 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750005Cleaning up temporary data from old regions at 1731521750032 (+27 ms)Running coprocessor post-open hooks at 1731521750063 (+31 ms)Region opened successfully at 1731521750063 2024-11-13T18:15:50,064 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 400b285d3ec5b0d43bcd37cdfa6b8f90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60034754, jitterRate=-0.10541245341300964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,064 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:15:50,064 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 400b285d3ec5b0d43bcd37cdfa6b8f90: Running coprocessor pre-open hook at 1731521750022Writing region info on filesystem at 1731521750022Initializing all the Stores at 1731521750023 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750024 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750025 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750025Cleaning up temporary data from old regions at 1731521750047 (+22 ms)Running coprocessor post-open hooks at 1731521750064 (+17 ms)Region opened successfully at 1731521750064 2024-11-13T18:15:50,065 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., pid=35, masterSystemTime=1731521749607 2024-11-13T18:15:50,066 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., pid=53, masterSystemTime=1731521749708 2024-11-13T18:15:50,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ae2919589ed9f92a8ad04abb8e24d5, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,068 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,069 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:50,069 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:15:50,069 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:50,070 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5aa68e8ac635bc9e7d6868f5f771281e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T18:15:50,070 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,070 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,070 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,070 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,072 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272983331631897f6cfeac3ce6bd7466 columnFamilyName colfamily11 2024-11-13T18:15:50,072 DEBUG [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,072 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(327): Store=272983331631897f6cfeac3ce6bd7466/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,072 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,074 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272983331631897f6cfeac3ce6bd7466 columnFamilyName colfamily21 2024-11-13T18:15:50,074 DEBUG [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,075 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(327): Store=272983331631897f6cfeac3ce6bd7466/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,075 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=17 2024-11-13T18:15:50,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 27ae2919589ed9f92a8ad04abb8e24d5, server=fda96cf025dc,35435,1731521744991 in 466 msec 2024-11-13T18:15:50,077 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:50,077 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:15:50,077 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:50,077 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8636f69311e6a46aa80948ea8be00961, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-13T18:15:50,078 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,078 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,078 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,078 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272983331631897f6cfeac3ce6bd7466 columnFamilyName colfamily31 2024-11-13T18:15:50,078 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,078 DEBUG [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=27ae2919589ed9f92a8ad04abb8e24d5, ASSIGN in 798 msec 2024-11-13T18:15:50,080 INFO [StoreOpener-272983331631897f6cfeac3ce6bd7466-1 {}] regionserver.HStore(327): Store=272983331631897f6cfeac3ce6bd7466/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,080 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,080 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:50,081 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,081 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,082 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,082 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,083 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,083 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,085 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,085 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily11 2024-11-13T18:15:50,085 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,086 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,086 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:50,088 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,089 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily21 2024-11-13T18:15:50,089 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:50,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37863ce38db650b86b62a2bccc5e7796, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:50,092 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5aa68e8ac635bc9e7d6868f5f771281e columnFamilyName colfamily11 2024-11-13T18:15:50,092 DEBUG [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,093 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,093 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(327): Store=5aa68e8ac635bc9e7d6868f5f771281e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,093 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,093 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,094 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,096 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 272983331631897f6cfeac3ce6bd7466; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72309310, jitterRate=0.07749268412590027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,096 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:15:50,096 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 272983331631897f6cfeac3ce6bd7466: Running coprocessor pre-open hook at 1731521750062Writing region info on filesystem at 1731521750062Initializing all the Stores at 1731521750067 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750067Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750068 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750068Cleaning up temporary data from old regions at 1731521750084 (+16 ms)Running coprocessor post-open hooks at 1731521750096 (+12 ms)Region opened successfully at 1731521750096 2024-11-13T18:15:50,096 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5aa68e8ac635bc9e7d6868f5f771281e columnFamilyName colfamily21 2024-11-13T18:15:50,096 DEBUG [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,097 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., pid=40, masterSystemTime=1731521749713 2024-11-13T18:15:50,098 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(327): Store=5aa68e8ac635bc9e7d6868f5f771281e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,098 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,100 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5aa68e8ac635bc9e7d6868f5f771281e columnFamilyName colfamily31 2024-11-13T18:15:50,100 DEBUG [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,101 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily31 2024-11-13T18:15:50,101 INFO [StoreOpener-5aa68e8ac635bc9e7d6868f5f771281e-1 {}] regionserver.HStore(327): Store=5aa68e8ac635bc9e7d6868f5f771281e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,101 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,101 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,102 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,102 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,103 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,103 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,104 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,104 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,106 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,106 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,107 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,107 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,107 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,108 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,110 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,111 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,116 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,117 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8636f69311e6a46aa80948ea8be00961; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61979771, jitterRate=-0.07642944157123566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,117 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:15:50,117 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8636f69311e6a46aa80948ea8be00961: Running coprocessor pre-open hook at 1731521750078Writing region info on filesystem at 1731521750078Initializing all the Stores at 1731521750080 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750080Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750082 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750082Cleaning up temporary data from old regions at 1731521750107 (+25 ms)Running coprocessor post-open hooks at 1731521750117 (+10 ms)Region opened successfully at 1731521750117 2024-11-13T18:15:50,118 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., pid=38, masterSystemTime=1731521749708 2024-11-13T18:15:50,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=28 2024-11-13T18:15:50,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,42475,1731521745166 in 422 msec 2024-11-13T18:15:50,120 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:50,120 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:15:50,120 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:50,120 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 38c892b7353daa24a9b6c917f1aa0a89, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-13T18:15:50,120 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,121 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,121 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,121 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,122 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:50,122 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:15:50,122 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:50,125 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:50,126 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 02893942b5e22ded04197e4e8486e0dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-13T18:15:50,126 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,126 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,126 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,127 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,127 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5aa68e8ac635bc9e7d6868f5f771281e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61466989, jitterRate=-0.08407048881053925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,127 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:15:50,127 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5aa68e8ac635bc9e7d6868f5f771281e: Running coprocessor pre-open hook at 1731521750070Writing region info on filesystem at 1731521750070Initializing all the Stores at 1731521750072 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750072Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750081 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750081Cleaning up temporary data from old regions at 1731521750106 (+25 ms)Running coprocessor post-open hooks at 1731521750127 (+21 ms)Region opened successfully at 1731521750127 2024-11-13T18:15:50,127 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,130 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., pid=34, masterSystemTime=1731521749607 2024-11-13T18:15:50,132 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38c892b7353daa24a9b6c917f1aa0a89 columnFamilyName colfamily11 2024-11-13T18:15:50,132 DEBUG [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272983331631897f6cfeac3ce6bd7466, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,133 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:50,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN in 851 msec 2024-11-13T18:15:50,135 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(327): Store=38c892b7353daa24a9b6c917f1aa0a89/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,135 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:50,135 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:15:50,135 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,136 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:50,136 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => fd437c88e4af73574ce0ce0882aca142, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-13T18:15:50,136 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,136 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,136 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,137 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:50,139 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily11 2024-11-13T18:15:50,139 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,140 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38c892b7353daa24a9b6c917f1aa0a89 columnFamilyName colfamily21 2024-11-13T18:15:50,140 DEBUG [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,141 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(327): Store=38c892b7353daa24a9b6c917f1aa0a89/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,141 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,141 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,141 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5aa68e8ac635bc9e7d6868f5f771281e, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:50,143 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38c892b7353daa24a9b6c917f1aa0a89 columnFamilyName colfamily31 2024-11-13T18:15:50,143 DEBUG [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,144 INFO [StoreOpener-38c892b7353daa24a9b6c917f1aa0a89-1 {}] regionserver.HStore(327): Store=38c892b7353daa24a9b6c917f1aa0a89/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,144 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,145 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,145 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,147 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,147 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,148 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=16 2024-11-13T18:15:50,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,42475,1731521745166 in 602 msec 2024-11-13T18:15:50,149 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,151 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily21 2024-11-13T18:15:50,151 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,151 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,152 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,153 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,156 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily31 2024-11-13T18:15:50,157 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN in 871 msec 2024-11-13T18:15:50,157 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,157 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,159 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,159 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd437c88e4af73574ce0ce0882aca142 columnFamilyName colfamily11 2024-11-13T18:15:50,159 DEBUG [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,159 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,159 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,46791,1731521745104, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:50,159 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,46791,1731521745104, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:50,160 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(327): Store=fd437c88e4af73574ce0ce0882aca142/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,160 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,161 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,161 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,162 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd437c88e4af73574ce0ce0882aca142 columnFamilyName colfamily21 2024-11-13T18:15:50,162 DEBUG [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,163 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,163 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(327): Store=fd437c88e4af73574ce0ce0882aca142/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,164 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,165 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,35435,1731521744991, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:50,166 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd437c88e4af73574ce0ce0882aca142 columnFamilyName colfamily31 2024-11-13T18:15:50,166 DEBUG [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,166 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,167 INFO [StoreOpener-fd437c88e4af73574ce0ce0882aca142-1 {}] regionserver.HStore(327): Store=fd437c88e4af73574ce0ce0882aca142/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,167 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,169 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,170 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=14 2024-11-13T18:15:50,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5aa68e8ac635bc9e7d6868f5f771281e, server=fda96cf025dc,46791,1731521745104 in 683 msec 2024-11-13T18:15:50,174 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,174 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5aa68e8ac635bc9e7d6868f5f771281e, ASSIGN in 893 msec 2024-11-13T18:15:50,175 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,176 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=29 2024-11-13T18:15:50,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 272983331631897f6cfeac3ce6bd7466, server=fda96cf025dc,35435,1731521744991 in 620 msec 2024-11-13T18:15:50,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=272983331631897f6cfeac3ce6bd7466, ASSIGN in 902 msec 2024-11-13T18:15:50,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=18 2024-11-13T18:15:50,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 37863ce38db650b86b62a2bccc5e7796, server=fda96cf025dc,46791,1731521745104 in 691 msec 2024-11-13T18:15:50,189 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,190 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 38c892b7353daa24a9b6c917f1aa0a89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73768979, jitterRate=0.09924344718456268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,190 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:15:50,190 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 38c892b7353daa24a9b6c917f1aa0a89: Running coprocessor pre-open hook at 1731521750121Writing region info on filesystem at 1731521750121Initializing all the Stores at 1731521750122 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750123 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750127 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750127Cleaning up temporary data from old regions at 1731521750147 (+20 ms)Running coprocessor post-open hooks at 1731521750190 (+43 ms)Region opened successfully at 1731521750190 2024-11-13T18:15:50,191 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89., pid=56, masterSystemTime=1731521749713 2024-11-13T18:15:50,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=37863ce38db650b86b62a2bccc5e7796, ASSIGN in 910 msec 2024-11-13T18:15:50,194 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,194 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:50,194 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 02893942b5e22ded04197e4e8486e0dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60216233, jitterRate=-0.10270820558071136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,195 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:50,195 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:15:50,195 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:50,195 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 02893942b5e22ded04197e4e8486e0dc: Running coprocessor pre-open hook at 1731521750127Writing region info on filesystem at 1731521750127Initializing all the Stores at 1731521750133 (+6 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750133Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750133Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750133Cleaning up temporary data from old regions at 1731521750161 (+28 ms)Running coprocessor post-open hooks at 1731521750195 (+34 ms)Region opened successfully at 1731521750195 2024-11-13T18:15:50,195 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 801d0f207e45efea50135a16f5fcbc99, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T18:15:50,195 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,195 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,196 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,196 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,197 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., pid=48, masterSystemTime=1731521749708 2024-11-13T18:15:50,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,202 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:50,202 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:50,203 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:50,203 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 64b5086eb7020ccba10769de2eaf3d9b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-13T18:15:50,203 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,203 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38c892b7353daa24a9b6c917f1aa0a89, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,204 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,204 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,210 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,211 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened fd437c88e4af73574ce0ce0882aca142; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72665208, jitterRate=0.08279597759246826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,211 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:15:50,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-13T18:15:50,211 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for fd437c88e4af73574ce0ce0882aca142: Running coprocessor pre-open hook at 1731521750137Writing region info on filesystem at 1731521750137Initializing all the Stores at 1731521750138 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750138Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750146 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750146Cleaning up temporary data from old regions at 1731521750174 (+28 ms)Running coprocessor post-open hooks at 1731521750211 (+37 ms)Region opened successfully at 1731521750211 2024-11-13T18:15:50,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 38c892b7353daa24a9b6c917f1aa0a89, server=fda96cf025dc,35435,1731521744991 in 507 msec 2024-11-13T18:15:50,213 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., pid=54, masterSystemTime=1731521749833 2024-11-13T18:15:50,218 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:50,218 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:50,218 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:50,218 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c39d6007aedf97e4c4a94511fe061ef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-13T18:15:50,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=38c892b7353daa24a9b6c917f1aa0a89, ASSIGN in 934 msec 2024-11-13T18:15:50,218 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,219 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,219 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,219 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:50,220 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,221 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,221 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,222 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 801d0f207e45efea50135a16f5fcbc99 columnFamilyName colfamily11 2024-11-13T18:15:50,222 DEBUG [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,223 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(327): Store=801d0f207e45efea50135a16f5fcbc99/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,223 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd437c88e4af73574ce0ce0882aca142, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:50,224 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c39d6007aedf97e4c4a94511fe061ef columnFamilyName colfamily11 2024-11-13T18:15:50,224 DEBUG [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,225 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily11 2024-11-13T18:15:50,225 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,226 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 801d0f207e45efea50135a16f5fcbc99 columnFamilyName colfamily21 2024-11-13T18:15:50,226 DEBUG [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,226 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(327): Store=5c39d6007aedf97e4c4a94511fe061ef/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,226 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,227 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(327): Store=801d0f207e45efea50135a16f5fcbc99/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:50,227 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,227 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,230 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 801d0f207e45efea50135a16f5fcbc99 columnFamilyName colfamily31 2024-11-13T18:15:50,230 DEBUG [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,230 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,231 INFO [StoreOpener-801d0f207e45efea50135a16f5fcbc99-1 {}] regionserver.HStore(327): Store=801d0f207e45efea50135a16f5fcbc99/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,231 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:50,232 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,233 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,233 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily21 2024-11-13T18:15:50,234 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,234 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,235 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,235 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c39d6007aedf97e4c4a94511fe061ef columnFamilyName colfamily21 2024-11-13T18:15:50,235 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,235 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,235 DEBUG [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,236 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,237 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(327): Store=5c39d6007aedf97e4c4a94511fe061ef/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,237 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,238 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily31 2024-11-13T18:15:50,238 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,238 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,239 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,239 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,240 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c39d6007aedf97e4c4a94511fe061ef columnFamilyName colfamily31 2024-11-13T18:15:50,240 DEBUG [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,240 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=7 2024-11-13T18:15:50,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,42475,1731521745166 in 606 msec 2024-11-13T18:15:50,241 INFO [StoreOpener-5c39d6007aedf97e4c4a94511fe061ef-1 {}] regionserver.HStore(327): Store=5c39d6007aedf97e4c4a94511fe061ef/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,241 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,241 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,242 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN in 962 msec 2024-11-13T18:15:50,243 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,244 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,244 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,245 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,245 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,245 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,246 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-13T18:15:50,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fd437c88e4af73574ce0ce0882aca142, server=fda96cf025dc,46791,1731521745104 in 565 msec 2024-11-13T18:15:50,248 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,248 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd437c88e4af73574ce0ce0882aca142, ASSIGN in 970 msec 2024-11-13T18:15:50,254 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,254 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 64b5086eb7020ccba10769de2eaf3d9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63889230, jitterRate=-0.047976285219192505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,254 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:15:50,254 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 64b5086eb7020ccba10769de2eaf3d9b: Running coprocessor pre-open hook at 1731521750204Writing region info on filesystem at 1731521750204Initializing all the Stores at 1731521750205 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750205Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750220 (+15 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750220Cleaning up temporary data from old regions at 1731521750244 (+24 ms)Running coprocessor post-open hooks at 1731521750254 (+10 ms)Region opened successfully at 1731521750254 2024-11-13T18:15:50,255 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,255 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 801d0f207e45efea50135a16f5fcbc99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73118382, jitterRate=0.08954879641532898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,255 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:15:50,255 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,256 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 801d0f207e45efea50135a16f5fcbc99: Running coprocessor pre-open hook at 1731521750196Writing region info on filesystem at 1731521750196Initializing all the Stores at 1731521750197 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750197Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750219 (+22 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750219Cleaning up temporary data from old regions at 1731521750235 (+16 ms)Running coprocessor post-open hooks at 1731521750255 (+20 ms)Region opened successfully at 1731521750255 2024-11-13T18:15:50,256 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 5c39d6007aedf97e4c4a94511fe061ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60518278, jitterRate=-0.09820738434791565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,256 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., pid=51, masterSystemTime=1731521749708 2024-11-13T18:15:50,256 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:15:50,257 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 5c39d6007aedf97e4c4a94511fe061ef: Running coprocessor pre-open hook at 1731521750219Writing region info on filesystem at 1731521750219Initializing all the Stores at 1731521750220 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750220Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750221 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750221Cleaning up temporary data from old regions at 1731521750245 (+24 ms)Running coprocessor post-open hooks at 1731521750256 (+11 ms)Region opened successfully at 1731521750256 2024-11-13T18:15:50,257 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., pid=43, masterSystemTime=1731521749713 2024-11-13T18:15:50,258 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., pid=55, masterSystemTime=1731521749833 2024-11-13T18:15:50,260 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:50,260 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:15:50,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,42475,1731521745166 2024-11-13T18:15:50,264 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:50,264 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:15:50,264 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:50,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,42475,1731521745166 because future has completed 2024-11-13T18:15:50,264 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 00a4d576f4c41497f18224865b73403a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-13T18:15:50,265 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,265 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,265 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,266 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,267 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:50,267 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:15:50,267 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:15:50,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 801d0f207e45efea50135a16f5fcbc99, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c39d6007aedf97e4c4a94511fe061ef, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:15:50,276 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,278 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a4d576f4c41497f18224865b73403a columnFamilyName colfamily11 2024-11-13T18:15:50,278 DEBUG [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,279 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(327): Store=00a4d576f4c41497f18224865b73403a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,279 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=11 2024-11-13T18:15:50,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 801d0f207e45efea50135a16f5fcbc99, server=fda96cf025dc,35435,1731521744991 in 691 msec 2024-11-13T18:15:50,281 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a4d576f4c41497f18224865b73403a columnFamilyName colfamily21 2024-11-13T18:15:50,281 DEBUG [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,282 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(327): Store=00a4d576f4c41497f18224865b73403a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,282 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=801d0f207e45efea50135a16f5fcbc99, ASSIGN in 1.0020 sec 2024-11-13T18:15:50,284 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a4d576f4c41497f18224865b73403a columnFamilyName colfamily31 2024-11-13T18:15:50,284 DEBUG [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,284 INFO [StoreOpener-00a4d576f4c41497f18224865b73403a-1 {}] regionserver.HStore(327): Store=00a4d576f4c41497f18224865b73403a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,285 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,286 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fda96cf025dc,42475,1731521745166, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:15:50,287 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,289 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=12 2024-11-13T18:15:50,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c39d6007aedf97e4c4a94511fe061ef, server=fda96cf025dc,46791,1731521745104 in 599 msec 2024-11-13T18:15:50,291 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,291 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,292 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5c39d6007aedf97e4c4a94511fe061ef, ASSIGN in 1.0120 sec 2024-11-13T18:15:50,293 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,297 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,298 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 00a4d576f4c41497f18224865b73403a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58965643, jitterRate=-0.1213434487581253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,298 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:15:50,298 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 00a4d576f4c41497f18224865b73403a: Running coprocessor pre-open hook at 1731521750266Writing region info on filesystem at 1731521750266Initializing all the Stores at 1731521750274 (+8 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750274Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750275 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750275Cleaning up temporary data from old regions at 1731521750291 (+16 ms)Running coprocessor post-open hooks at 1731521750298 (+7 ms)Region opened successfully at 1731521750298 2024-11-13T18:15:50,299 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., pid=42, masterSystemTime=1731521749713 2024-11-13T18:15:50,302 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:50,302 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:15:50,302 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:50,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => dae4cf552a1afaf0b041668499d99e20, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-13T18:15:50,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:15:50,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,303 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-13T18:15:50,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,42475,1731521745166 in 637 msec 2024-11-13T18:15:50,305 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN in 1.0260 sec 2024-11-13T18:15:50,307 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dae4cf552a1afaf0b041668499d99e20 columnFamilyName colfamily11 2024-11-13T18:15:50,307 DEBUG [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,308 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(327): Store=dae4cf552a1afaf0b041668499d99e20/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,308 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=00a4d576f4c41497f18224865b73403a, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a4d576f4c41497f18224865b73403a, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,313 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dae4cf552a1afaf0b041668499d99e20 columnFamilyName colfamily21 2024-11-13T18:15:50,313 DEBUG [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,315 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(327): Store=dae4cf552a1afaf0b041668499d99e20/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,315 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=8 2024-11-13T18:15:50,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 00a4d576f4c41497f18224865b73403a, server=fda96cf025dc,35435,1731521744991 in 735 msec 2024-11-13T18:15:50,317 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dae4cf552a1afaf0b041668499d99e20 columnFamilyName colfamily31 2024-11-13T18:15:50,317 DEBUG [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:15:50,317 INFO [StoreOpener-dae4cf552a1afaf0b041668499d99e20-1 {}] regionserver.HStore(327): Store=dae4cf552a1afaf0b041668499d99e20/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:15:50,318 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00a4d576f4c41497f18224865b73403a, ASSIGN in 1.0390 sec 2024-11-13T18:15:50,319 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,319 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,320 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,320 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,321 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:15:50,322 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,325 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:15:50,325 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened dae4cf552a1afaf0b041668499d99e20; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71179194, jitterRate=0.06065264344215393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:15:50,325 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:15:50,325 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for dae4cf552a1afaf0b041668499d99e20: Running coprocessor pre-open hook at 1731521750304Writing region info on filesystem at 1731521750304Initializing all the Stores at 1731521750305 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750305Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750305Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521750305Cleaning up temporary data from old regions at 1731521750320 (+15 ms)Running coprocessor post-open hooks at 1731521750325 (+5 ms)Region opened successfully at 1731521750325 2024-11-13T18:15:50,326 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., pid=46, masterSystemTime=1731521749713 2024-11-13T18:15:50,329 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:50,329 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:15:50,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:15:50,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure dae4cf552a1afaf0b041668499d99e20, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:15:50,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=21 2024-11-13T18:15:50,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure dae4cf552a1afaf0b041668499d99e20, server=fda96cf025dc,35435,1731521744991 in 724 msec 2024-11-13T18:15:50,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=4 2024-11-13T18:15:50,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dae4cf552a1afaf0b041668499d99e20, ASSIGN in 1.0660 sec 2024-11-13T18:15:50,350 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:15:50,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731521750350"}]},"ts":"1731521750350"} 2024-11-13T18:15:50,355 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-13T18:15:50,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:15:50,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.8420 sec 2024-11-13T18:15:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:15:50,748 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-13T18:15:50,748 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-13T18:15:50,749 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:15:50,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42475 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T18:15:50,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-13T18:15:50,773 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:15:50,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-13T18:15:50,827 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,835 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,843 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,844 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,845 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:50,953 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:50,955 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:50,962 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,046 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,133 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,139 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,141 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,143 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,145 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,146 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,150 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,225 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,227 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,233 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,238 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,240 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eek', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,248 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eel', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,304 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,308 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,309 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,311 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,313 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:15:51,383 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,385 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,457 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,465 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,478 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,483 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,485 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hho', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,525 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,528 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,531 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,539 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,541 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,627 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,630 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,633 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,636 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,722 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,735 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,790 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,797 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,798 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,803 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,805 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lls', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,807 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:51,854 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,860 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,864 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,867 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:51,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,961 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,033 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,036 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,129 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,133 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,135 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,136 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,138 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,202 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,206 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,222 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,225 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,271 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,293 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,295 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,297 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,299 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,301 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,337 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,344 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,351 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,354 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,403 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,406 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,408 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,414 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,416 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,464 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,473 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,475 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,478 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,522 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,540 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,543 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,545 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,548 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,550 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,597 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,654 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,657 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,661 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,666 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,668 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,670 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,671 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xye', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2] 2024-11-13T18:15:52,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,702 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,706 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., hostname=fda96cf025dc,35435,1731521744991, seqNum=2] 2024-11-13T18:15:52,751 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677., hostname=fda96cf025dc,46791,1731521745104, seqNum=2] 2024-11-13T18:15:52,781 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:15:52,800 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:15:52,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:15:52,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:15:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35435 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:15:52,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46791 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:15:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:15:53,512 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:15:53,653 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-13T18:15:53,658 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:15:54,137 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-13T18:15:54,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42475 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T18:15:54,414 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-13T18:15:54,415 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER fda96cf025dc,42475,1731521745166 2024-11-13T18:15:54,415 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing fda96cf025dc,42475,1731521745166 2024-11-13T18:15:54,416 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server fda96cf025dc,42475,1731521745166: Simulated kill ***** 2024-11-13T18:15:54,416 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T18:15:54,419 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T18:15:54,420 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T18:15:54,423 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T18:15:54,424 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T18:15:54,431 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 619497216 }, "NonHeapMemoryUsage": { "committed": 158400512, "init": 7667712, "max": -1, "used": 155509800 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 1501001, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 306, "ProcessCallTime_min": 0, "ProcessCallTime_max": 710, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 177, "ProcessCallTime_median": 355, "ProcessCallTime_75th_percentile": 532, "ProcessCallTime_90th_percentile": 639, "ProcessCallTime_95th_percentile": 674, "ProcessCallTime_98th_percentile": 695, "ProcessCallTime_99th_percentile": 702, "ProcessCallTime_99.9th_percentile": 709, "ProcessCallTime_TimeRangeCount_0-1": 306, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 306, "QueueCallTime_min": 0, "QueueCallTime_max": 336, "QueueCallTime_mean": 4, "QueueCallTime_25th_percentile": 84, "QueueCallTime_median": 168, "QueueCallTime_75th_percentile": 252, "QueueCallTime_90th_percentile": 302, "QueueCallTime_95th_percentile": 319, "QueueCallTime_98th_percentile": 329, "QueueCallTime_99th_percentile": 332, "QueueCallTime_99.9th_percentile": 335, "QueueCallTime_TimeRangeCount_0-1": 306, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 306, "TotalCallTime_min": 0, "TotalCallTime_max": 748, "TotalCallTime_mean": 11, "TotalCallTime_25th_percentile": 187, "TotalCallTime_median": 374, "TotalCallTime_75th_percentile": 561, "TotalCallTime_90th_percentile": 673, "TotalCallTime_95th_percentile": 710, "TotalCallTime_98th_percentile": 733, "TotalCallTime_99th_percentile": 740, "TotalCallTime_99.9th_percentile": 747, "TotalCallTime_TimeRangeCount_0-1": 306, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 306, "ResponseSize_min": 0, "ResponseSize_max": 45021, "ResponseSize_mean": 5830, "ResponseSize_25th_percentile": 11255, "ResponseSize_median": 22510, "ResponseSize_75th_percentile": 33765, "ResponseSize_90th_percentile": 40518, "ResponseSize_95th_percentile": 42769, "ResponseSize_98th_percentile": 44120, "ResponseSize_99th_percentile": 44570, "ResponseSize_99.9th_percentile": 44975, "ResponseSize_SizeRangeCount_0-10": 306, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 306, "RequestSize_min": 42, "RequestSize_max": 508072, "RequestSize_mean": 4892, "RequestSize_25th_percentile": 127049, "RequestSize_median": 254057, "RequestSize_75th_percentile": 381064, "RequestSize_90th_percentile": 457269, "RequestSize_95th_percentile": 482670, "RequestSize_98th_percentile": 497911, "RequestSize_99th_percentile": 502991, "RequestSize_99.9th_percentile": 507563, "RequestSize_SizeRangeCount_0-10": 306, "sentBytes": 1506690 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55845", "tag.serverName": "fda96cf025dc,46791,1731521745104", "tag.clusterId": "7bfa8d2f-88be-48ac-b35b-b39c6e6cb643", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "regionCount": 9, "storeCount": 27, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 6912, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731521745104, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 1, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 9, "PutBatch_min": 3, "PutBatch_max": 102, "PutBatch_mean": 63, "PutBatch_25th_percentile": 27, "PutBatch_median": 52, "PutBatch_75th_percentile": 77, "PutBatch_90th_percentile": 92, "PutBatch_95th_percentile": 97, "PutBatch_98th_percentile": 100, "PutBatch_99th_percentile": 101, "PutBatch_99.9th_percentile": 101, "PutBatch_TimeRangeCount_0-1": 9, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T18:15:54,436 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41233 {}] master.MasterRpcServices(700): fda96cf025dc,42475,1731521745166 reported a fatal error: ***** ABORTING region server fda96cf025dc,42475,1731521745166: Simulated kill ***** 2024-11-13T18:15:54,439 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,42475,1731521745166' ***** 2024-11-13T18:15:54,439 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T18:15:54,440 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:15:54,440 INFO [RS:2;fda96cf025dc:42475 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T18:15:54,440 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:15:54,440 INFO [RS:2;fda96cf025dc:42475 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T18:15:54,440 INFO [RS:2;fda96cf025dc:42475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:15:54,441 INFO [RS:2;fda96cf025dc:42475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fda96cf025dc:42475. 2024-11-13T18:15:54,441 DEBUG [RS:2;fda96cf025dc:42475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:15:54,446 DEBUG [RS:2;fda96cf025dc:42475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:54,447 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:15:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741836_1012 (size=54214) 2024-11-13T18:15:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741836_1012 (size=54214) 2024-11-13T18:15:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741836_1012 (size=54214) 2024-11-13T18:15:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741835_1011 (size=297717) 2024-11-13T18:15:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741835_1011 (size=297717) 2024-11-13T18:15:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741835_1011 (size=297717) 2024-11-13T18:15:54,461 DEBUG [RS:2;fda96cf025dc:42475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:15:54,461 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:15:54,461 INFO [RS:2;fda96cf025dc:42475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42475 2024-11-13T18:15:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:15:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,42475,1731521745166 2024-11-13T18:15:54,470 INFO [RS:2;fda96cf025dc:42475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:15:54,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,42475,1731521745166] 2024-11-13T18:15:54,474 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,42475,1731521745166 already deleted, retry=false 2024-11-13T18:15:54,474 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fda96cf025dc,42475,1731521745166 on fda96cf025dc,41233,1731521744053 2024-11-13T18:15:54,479 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,483 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for fda96cf025dc,42475,1731521745166 (carryingMeta=true) fda96cf025dc,42475,1731521745166/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@218f2c16[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T18:15:54,484 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:15:54,484 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:15:54,485 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:54,501 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:15:54,541 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:15:54,543 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,544 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:15:54,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:15:54,544 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:15:54,545 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:15:54,546 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:15:54,546 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:15:54,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:15:54,546 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:15:54,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:15:54,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:15:54,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:15:54,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:15:54,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:15:54,553 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T18:15:54,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:15:54,555 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,42475,1731521745166, table=hbase:meta, region=1588230740 2024-11-13T18:15:54,555 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPEN 2024-11-13T18:15:54,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:54,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:54,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:54,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:54,560 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521784560 2024-11-13T18:15:54,560 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:15:54,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:15:54,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:15:54,561 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,561 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:15:54,562 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:54,564 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true, isMeta: true 2024-11-13T18:15:54,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:54,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:15:54,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:54,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:54,567 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting 2024-11-13T18:15:54,568 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=1, meta=true 2024-11-13T18:15:54,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta}] 2024-11-13T18:15:54,571 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:15:54,571 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:15:54,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:15:54,572 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:15:54,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:54,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42475-0x1003a5b1ab20003, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:15:54,574 INFO [RS:2;fda96cf025dc:42475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:15:54,574 INFO [RS:2;fda96cf025dc:42475 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,42475,1731521745166; zookeeper connection closed. 2024-11-13T18:15:54,574 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77dbe6aa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77dbe6aa 2024-11-13T18:15:54,636 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:15:54,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T18:15:54,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:15:54,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-13T18:15:54,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:15:54,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T18:15:54,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:15:54,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:15:54,676 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:54,677 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:54,678 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:15:54,678 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:54,678 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:54,678 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:54,886 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:54,886 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:54,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:15:54,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:54,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:54,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:55,196 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:55,196 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:55,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:15:55,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:55,213 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:15:55,214 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:15:55,706 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:55,707 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:55,707 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:15:55,708 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55,708 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55,708 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:56,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:56,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:56,717 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:56,721 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2157 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:15:56,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:56,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:15:56,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:56,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:58,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:15:58,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:15:58,747 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:15:58,747 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4186 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:15:58,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:58,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:15:58,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:15:58,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:02,756 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:02,756 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:02,757 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:02,758 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8197 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:02,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:02,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:02,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:02,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:12,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:16:12,855 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:12,856 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:12,857 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:12,857 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18296 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:12,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:12,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:12,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:12,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:19,684 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fda96cf025dc:41233 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T18:16:22,955 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:22,956 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:22,957 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:22,957 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28396 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:22,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:22,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:22,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:22,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:28,348 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T18:16:28,348 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T18:16:33,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:33,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:33,057 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:33,058 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38497 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:33,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:33,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:33,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:33,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:38,756 WARN [RS:1;fda96cf025dc:46791 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fda96cf025dc,42475,1731521745166: status=status unset, state=RUNNING, startTime=1731521754484, completionTime=-1 appears to have been leaked 2024-11-13T18:16:42,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:16:43,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:43,136 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:43,137 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:43,137 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48576 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:43,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:43,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:43,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:43,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:48,281 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 38c892b7353daa24a9b6c917f1aa0a89 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,281 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 00a4d576f4c41497f18224865b73403a changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,282 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 801d0f207e45efea50135a16f5fcbc99 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,282 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e5bab85a9454bcf58ade3574335c3bd1 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,282 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 27ae2919589ed9f92a8ad04abb8e24d5 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,282 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dae4cf552a1afaf0b041668499d99e20 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1b91248305f9bd170badfc1ac16c2f2b changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b196a2f44f4f468f8bb9a2e59b301747 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 272983331631897f6cfeac3ce6bd7466 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fd437c88e4af73574ce0ce0882aca142 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1e715c3850b0001604b62543d4e17908 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5c39d6007aedf97e4c4a94511fe061ef changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5aa68e8ac635bc9e7d6868f5f771281e changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 37863ce38db650b86b62a2bccc5e7796 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 575404eded2e7a15cbfa9186eea1d1b1 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 644025ac5ad44a7ca4a26c89d6611d8b changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5a59239e975bc7ad59b0ae9308416d51 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:48,283 DEBUG [master/fda96cf025dc:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cf5aa9737456e8265a9b1e93d818d677 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:16:50,685 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fda96cf025dc:41233 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T18:16:52,206 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T18:16:54,566 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T18:15:54.566Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:54.678Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:54.887Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55.197Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:55.708Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:15:56.722Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:15:58.748Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:02.759Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:12.859Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:22.958Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:33.058Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:43.138Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T18:16:54,568 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:16:54,568 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:16:54,568 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:16:54,568 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:16:54,569 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:16:54,569 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:16:54,569 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:16:54,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:16:54,569 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:16:54,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T18:16:54,573 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:16:54,574 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:16:54,575 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:16:54,575 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:16:54,575 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:16:54,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T18:16:54,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:16:54,576 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,42475,1731521745166, table=hbase:meta, region=1588230740 2024-11-13T18:16:54,577 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPEN 2024-11-13T18:16:54,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:16:54,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:16:54,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:16:54,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:16:54,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:16:54,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:16:54,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521844584 2024-11-13T18:16:54,585 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:16:54,585 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true, isMeta: true 2024-11-13T18:16:54,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:54,585 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:54,586 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:54,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:54,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:54,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:54,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:54,595 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=1, meta=true 2024-11-13T18:16:54,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta}] 2024-11-13T18:16:54,596 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:16:54,602 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:16:54,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:16:54,696 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:54,696 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:54,697 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:16:54,697 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:54,697 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:54,697 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:54,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46791 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-13T18:16:54,805 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, size=52.9 K (54214bytes) 2024-11-13T18:16:54,806 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T18:16:54,806 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T18:16:54,806 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta 2024-11-13T18:16:54,814 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta after 5ms 2024-11-13T18:16:54,824 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta took 21ms 2024-11-13T18:16:54,834 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:16:54,843 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp 2024-11-13T18:16:54,847 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta so closing down 2024-11-13T18:16:54,847 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:16:54,849 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp 2024-11-13T18:16:54,851 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:16:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741864_1040 (size=53973) 2024-11-13T18:16:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741864_1040 (size=53973) 2024-11-13T18:16:54,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741864_1040 (size=53973) 2024-11-13T18:16:54,867 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp (wrote 57 edits, skipped 0 edits in 1 ms) 2024-11-13T18:16:54,869 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T18:16:54,873 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 44 ms; skipped=1; WAL=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-13T18:16:54,873 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, journal: Splitting hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, size=52.9 K (54214bytes) at 1731521814805Finishing writing output for hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta so closing down at 1731521814847 (+42 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp at 1731521814849 (+2 ms)3 split writer threads finished at 1731521814852 (+3 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp (wrote 57 edits, skipped 0 edits in 1 ms) at 1731521814867 (+15 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731521814869 (+2 ms)Processed 58 edits across 1 Regions in 44 ms; skipped=1; WAL=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731521814873 (+4 ms) 2024-11-13T18:16:54,874 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta 2024-11-13T18:16:54,875 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T18:16:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T18:16:54,884 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs 2024-11-13T18:16:54,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T18:16:54,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 in 280 msec 2024-11-13T18:16:54,888 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:16:54,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-13T18:16:54,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 in 293 msec 2024-11-13T18:16:54,893 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting dir is empty, no logs to split. 2024-11-13T18:16:54,893 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=0, meta=true 2024-11-13T18:16:54,893 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,42475,1731521745166 WAL splitting is done? wals=0, meta=true 2024-11-13T18:16:54,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:16:54,893 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:16:54,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:16:54,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:54,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:55,216 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:55,216 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:55,216 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:16:55,217 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55,217 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55,217 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:55,725 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:55,725 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:55,726 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:16:55,726 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55,726 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55,726 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:56,735 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:56,736 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:56,737 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:56,737 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2152 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:58,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:16:58,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:16:58,747 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:16:58,748 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4163 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:16:58,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:58,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:16:58,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:16:58,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:02,765 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:02,765 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:02,766 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:02,767 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8182 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:02,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:02,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:02,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:02,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:12,796 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:12,796 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:12,797 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:12,798 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18212 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:12,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:12,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:12,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:12,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:12,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:17:22,856 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:22,856 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:22,857 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:22,858 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28272 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:22,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:22,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:22,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:22,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:32,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:32,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:32,947 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:32,948 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38362 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:32,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:32,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:32,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:42,806 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:17:43,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:43,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:43,027 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:43,028 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48442 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:43,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:43,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:43,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:43,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:47,183 WARN [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 20, running: 0 2024-11-13T18:17:47,515 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-13T18:17:54,586 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T18:16:54.587Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:54.697Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:54.907Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55.217Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:55.726Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:16:56.738Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:16:58.749Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:02.768Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:12.798Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:22.858Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:32.948Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:43.028Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T18:17:54,587 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:17:54,587 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:17:54,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:17:54,587 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:17:54,588 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:17:54,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:17:54,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:17:54,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:17:54,589 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:17:54,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-13T18:17:54,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:17:54,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:17:54,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:17:54,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:17:54,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:17:54,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:17:54,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:17:54,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:17:54,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:17:54,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:17:54,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:17:54,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T18:17:54,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:17:54,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,42475,1731521745166, table=hbase:meta, region=1588230740 2024-11-13T18:17:54,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPEN 2024-11-13T18:17:54,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:17:54,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:17:54,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:17:54,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:17:54,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:17:54,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:17:54,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521904603 2024-11-13T18:17:54,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:17:54,604 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:54,604 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:54,605 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:54,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:54,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:54,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:54,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:54,613 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting dir is empty, no logs to split. 2024-11-13T18:17:54,613 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=0, meta=true 2024-11-13T18:17:54,613 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,42475,1731521745166 WAL splitting is done? wals=0, meta=true 2024-11-13T18:17:54,613 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:17:54,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:17:54,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:17:54,618 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:17:54,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:17:54,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:17:54,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:54,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:54,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:17:54,717 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:54,717 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:54,717 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:54,926 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:54,926 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:54,926 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:17:54,926 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:54,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:54,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:55,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:55,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:55,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:17:55,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:55,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:55,746 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:55,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:17:55,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:56,755 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:56,756 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:56,758 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:56,758 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:56,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:56,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:56,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:56,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:57,687 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fda96cf025dc:41233 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T18:17:58,765 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:17:58,765 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:17:58,766 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:17:58,767 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4163 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:17:58,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:58,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:17:58,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:17:58,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:02,786 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:02,786 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:02,787 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:02,788 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8184 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:18:02,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:02,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:02,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:02,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:12,806 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:18:12,896 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:12,896 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:12,897 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:12,897 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18293 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:18:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:22,945 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:22,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:22,947 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:22,947 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28343 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:18:22,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:22,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:22,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:22,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:32,986 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:32,986 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:32,987 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:32,987 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38383 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:18:32,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:32,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:32,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:32,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:40,871 WARN [RS:0;fda96cf025dc:35435 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fda96cf025dc,42475,1731521745166: status=Processing ServerCrashProcedure of fda96cf025dc,42475,1731521745166 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731521754553, completionTime=-1 appears to have been leaked 2024-11-13T18:18:40,872 WARN [RS:0;fda96cf025dc:35435 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fda96cf025dc,42475,1731521745166: status=Processing ServerCrashProcedure of fda96cf025dc,42475,1731521745166 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731521814575, completionTime=-1 appears to have been leaked 2024-11-13T18:18:42,806 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:18:43,005 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:43,006 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:43,007 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:43,007 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda96cf025dc,42475,1731521745166 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48403 ms java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T18:18:43,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:43,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:43,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:43,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:54,606 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T18:17:54.608Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:54.717Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:54.927Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55.237Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:55.747Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:17:56.759Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:17:58.767Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:02.788Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:12.898Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:22.947Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:32.988Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:43.008Z, java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T18:18:54,608 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:54,608 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:54,608 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:54,608 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:54,609 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:54,609 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:54,609 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:54,609 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:54,609 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:54,611 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41233 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:54,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-13T18:18:54,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:54,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:54,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:54,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:54,615 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:54,615 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T18:18:54,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:54,620 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,42475,1731521745166, table=hbase:meta, region=1588230740 2024-11-13T18:18:54,621 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,42475,1731521745166, state=OPEN 2024-11-13T18:18:54,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,624 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521964624 2024-11-13T18:18:54,624 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:54,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:18:54,625 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:54,625 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:54,625 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:54,626 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:54,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:54,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:54,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T18:18:54,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:54,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:54,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:54,736 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:54,736 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1] 2024-11-13T18:18:54,737 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:18:54,737 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:18:54,737 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:18:54,737 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:18:54,778 DEBUG [fda96cf025dc:41233 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T18:18:54,778 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:18:54,778 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:18:54,778 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:18:54,779 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:18:54,779 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:18:54,779 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:18:54,779 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:18:54,779 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:18:54,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:54,780 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPENING 2024-11-13T18:18:54,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:54,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,783 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:54,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:54,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:18:54,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:18:54,944 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:18:54,944 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:18:54,944 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:18:54,945 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:54,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1] 2024-11-13T18:18:54,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:18:54,947 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C35435%2C1731521744991.meta, suffix=.meta, logDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991, archiveDir=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs, maxLogs=32 2024-11-13T18:18:54,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:18:54,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35435 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:39815 deadline: 1731521994627, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 2024-11-13T18:18:54,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:18:54,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:18:54,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 from cache 2024-11-13T18:18:54,965 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991/fda96cf025dc%2C35435%2C1731521744991.meta.1731521934948.meta, exclude list is [], retry=0 2024-11-13T18:18:54,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37435,DS-2bb65705-56f0-4646-a633-f8a690c0d7ae,DISK] 2024-11-13T18:18:54,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38349,DS-5b711bb7-666b-4bd4-bcca-31f2a817ede8,DISK] 2024-11-13T18:18:54,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34407,DS-75a99b11-9703-4615-ac8d-dc275ad3ddeb,DISK] 2024-11-13T18:18:54,972 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991/fda96cf025dc%2C35435%2C1731521744991.meta.1731521934948.meta 2024-11-13T18:18:54,973 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32783:32783),(127.0.0.1/127.0.0.1:33047:33047),(127.0.0.1/127.0.0.1:41085:41085)] 2024-11-13T18:18:54,973 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:18:54,973 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:18:54,974 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:18:54,974 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:18:54,974 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:18:54,974 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:54,974 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:18:54,974 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:18:54,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:18:54,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:18:54,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:54,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:18:54,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:18:54,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:18:54,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:54,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:18:54,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:18:54,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:18:54,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:54,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:18:54,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:18:54,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:18:54,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:54,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:18:54,984 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:18:54,985 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:18:54,987 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740 2024-11-13T18:18:54,990 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T18:18:55,006 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T18:18:55,010 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-13T18:18:55,098 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/ee88a9ea77af4451a862600b7f7d49aa is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89./info:regioninfo/1731521750198/Put/seqid=0 2024-11-13T18:18:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741866_1042 (size=58053) 2024-11-13T18:18:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741866_1042 (size=58053) 2024-11-13T18:18:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741866_1042 (size=58053) 2024-11-13T18:18:55,127 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/ee88a9ea77af4451a862600b7f7d49aa 2024-11-13T18:18:55,198 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/ns/ead56e756e814d11bc47cfc8f7a18abf is 43, key is default/ns:d/1731521748153/Put/seqid=0 2024-11-13T18:18:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741867_1043 (size=5153) 2024-11-13T18:18:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741867_1043 (size=5153) 2024-11-13T18:18:55,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741867_1043 (size=5153) 2024-11-13T18:18:55,208 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/ns/ead56e756e814d11bc47cfc8f7a18abf 2024-11-13T18:18:55,232 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/table/338c6ef8b43040ae90fd70ac8a3b7705 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731521750350/Put/seqid=0 2024-11-13T18:18:55,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741868_1044 (size=5620) 2024-11-13T18:18:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741868_1044 (size=5620) 2024-11-13T18:18:55,243 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/table/338c6ef8b43040ae90fd70ac8a3b7705 2024-11-13T18:18:55,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741868_1044 (size=5620) 2024-11-13T18:18:55,252 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/ee88a9ea77af4451a862600b7f7d49aa as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/info/ee88a9ea77af4451a862600b7f7d49aa 2024-11-13T18:18:55,256 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:55,256 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1] 2024-11-13T18:18:55,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35435 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:39815 deadline: 1731521994625, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 2024-11-13T18:18:55,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:18:55,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fda96cf025dc,35435,1731521744991 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:18:55,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1 from cache 2024-11-13T18:18:55,259 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/info/ee88a9ea77af4451a862600b7f7d49aa, entries=260, sequenceid=60, filesize=56.7 K 2024-11-13T18:18:55,262 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/ns/ead56e756e814d11bc47cfc8f7a18abf as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/ns/ead56e756e814d11bc47cfc8f7a18abf 2024-11-13T18:18:55,273 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/ns/ead56e756e814d11bc47cfc8f7a18abf, entries=2, sequenceid=60, filesize=5.0 K 2024-11-13T18:18:55,275 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/table/338c6ef8b43040ae90fd70ac8a3b7705 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/table/338c6ef8b43040ae90fd70ac8a3b7705 2024-11-13T18:18:55,293 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/table/338c6ef8b43040ae90fd70ac8a3b7705, entries=2, sequenceid=60, filesize=5.5 K 2024-11-13T18:18:55,299 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 287ms, sequenceid=60, compaction requested=false; wal=null 2024-11-13T18:18:55,301 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T18:18:55,306 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:18:55,306 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:18:55,307 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:18:55,310 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:18:55,321 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-13T18:18:55,324 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72200593, jitterRate=0.07587267458438873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:18:55,325 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:18:55,325 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731521934974Writing region info on filesystem at 1731521934974Initializing all the Stores at 1731521934976 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521934976Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521934976Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521934976Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731521934976Obtaining lock to block concurrent updates at 1731521935010 (+34 ms)Preparing flush snapshotting stores in 1588230740 at 1731521935010Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731521935014 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731521935014Flushing 1588230740/info: creating writer at 1731521935018 (+4 ms)Flushing 1588230740/info: appending metadata at 1731521935090 (+72 ms)Flushing 1588230740/info: closing flushed file at 1731521935093 (+3 ms)Flushing 1588230740/ns: creating writer at 1731521935180 (+87 ms)Flushing 1588230740/ns: appending metadata at 1731521935197 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731521935197Flushing 1588230740/table: creating writer at 1731521935215 (+18 ms)Flushing 1588230740/table: appending metadata at 1731521935232 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731521935232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77cccf66: reopening flushed file at 1731521935251 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5853e1b5: reopening flushed file at 1731521935261 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d673879: reopening flushed file at 1731521935273 (+12 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 287ms, sequenceid=60, compaction requested=false; wal=null at 1731521935299 (+26 ms)Cleaning up temporary data from old regions at 1731521935306 (+7 ms)Running coprocessor post-open hooks at 1731521935325 (+19 ms)Region opened successfully at 1731521935325 2024-11-13T18:18:55,329 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731521934935 2024-11-13T18:18:55,334 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:18:55,334 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:18:55,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:55,336 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:55,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,340 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:55,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T18:18:55,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 in 557 msec 2024-11-13T18:18:55,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-13T18:18:55,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 728 msec 2024-11-13T18:18:55,766 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:18:55,766 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1] 2024-11-13T18:18:55,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:55,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521995809 2024-11-13T18:18:55,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522055809 2024-11-13T18:18:55,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.1840 sec 2024-11-13T18:18:55,810 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fda96cf025dc,42475,1731521745166 had 9 regions 2024-11-13T18:18:55,810 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:18:55,810 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:55,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:55,811 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:55,811 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:55,811 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:55,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:55,811 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:55,811 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:55,811 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:55,811 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:55,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:55,812 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:55,813 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:18:55,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:55,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:55,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:55,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:55,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:55,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:55,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:55,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:55,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T18:18:55,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=61 2024-11-13T18:18:55,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,35435,1731521744991, table=hbase:meta, region=1588230740 2024-11-13T18:18:55,821 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:55,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:55,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:55,833 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521965833 2024-11-13T18:18:55,833 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:55,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:55,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:55,844 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:55,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521995845 2024-11-13T18:18:55,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522055845 2024-11-13T18:18:55,845 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-13T18:18:55,846 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fda96cf025dc,42475,1731521745166 had 8 regions 2024-11-13T18:18:55,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:55,848 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true, isMeta: false 2024-11-13T18:18:55,853 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=1, meta=false 2024-11-13T18:18:55,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395}] 2024-11-13T18:18:55,853 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:18:55,853 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:55,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:55,853 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:56,095 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:56,095 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:56,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:56,095 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:56,096 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:56,096 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:56,096 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:56,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:56,096 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:56,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-11-13T18:18:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:56,107 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:56,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:56,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:56,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:56,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T18:18:56,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=61 2024-11-13T18:18:56,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,35435,1731521744991, table=hbase:meta, region=1588230740 2024-11-13T18:18:56,110 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:56,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:56,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:56,113 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:56,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:56,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:56,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:56,129 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521966129 2024-11-13T18:18:56,129 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:56,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:56,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:56,148 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:56,148 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521996148 2024-11-13T18:18:56,148 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522056148 2024-11-13T18:18:56,148 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-13T18:18:56,148 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true, isMeta: false 2024-11-13T18:18:56,161 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=1, meta=false 2024-11-13T18:18:56,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395}] 2024-11-13T18:18:56,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:56,169 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:56,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:18:56,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41627, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:18:56,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46791 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-13T18:18:56,360 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395, size=290.7 K (297717bytes) 2024-11-13T18:18:56,360 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 2024-11-13T18:18:56,362 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 after 2ms 2024-11-13T18:18:56,374 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 took 14ms 2024-11-13T18:18:56,404 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3eefcb355d011c4ea65b0572629d5ef5: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,411 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 09ff4ab7a9d0091782b93c194b270c43: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,418 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fb720a25eb86c541f9c1f814843c651b: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,421 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7fb2e398d735ceeb281622c9b30d4cf1: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,423 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 400b285d3ec5b0d43bcd37cdfa6b8f90: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,435 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8636f69311e6a46aa80948ea8be00961: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,440 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 02893942b5e22ded04197e4e8486e0dc: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,454 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 64b5086eb7020ccba10769de2eaf3d9b: last_flushed_sequence_id: 18446744073709551615 2024-11-13T18:18:56,462 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,465 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,481 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,481 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 so closing down 2024-11-13T18:18:56,481 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:18:56,489 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,490 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,490 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,493 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,497 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,498 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,500 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,507 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,509 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,528 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,532 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,537 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,548 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp 2024-11-13T18:18:56,549 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:18:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741870_1046 (size=36767) 2024-11-13T18:18:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741870_1046 (size=36767) 2024-11-13T18:18:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741870_1046 (size=36767) 2024-11-13T18:18:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741871_1047 (size=36767) 2024-11-13T18:18:56,714 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:18:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741871_1047 (size=36767) 2024-11-13T18:18:56,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741869_1045 (size=36767) 2024-11-13T18:18:56,718 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004 2024-11-13T18:18:56,719 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:18:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741869_1045 (size=36767) 2024-11-13T18:18:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741871_1047 (size=36767) 2024-11-13T18:18:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741869_1045 (size=36767) 2024-11-13T18:18:56,724 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004 2024-11-13T18:18:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741872_1048 (size=36767) 2024-11-13T18:18:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741872_1048 (size=36767) 2024-11-13T18:18:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741872_1048 (size=36767) 2024-11-13T18:18:56,801 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 2 ms) 2024-11-13T18:18:56,809 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004 2024-11-13T18:18:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741873_1049 (size=36767) 2024-11-13T18:18:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741873_1049 (size=36767) 2024-11-13T18:18:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741873_1049 (size=36767) 2024-11-13T18:18:56,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T18:18:56,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741874_1050 (size=36767) 2024-11-13T18:18:56,852 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T18:18:56,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741874_1050 (size=36767) 2024-11-13T18:18:56,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741874_1050 (size=36767) 2024-11-13T18:18:56,858 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004 2024-11-13T18:18:56,860 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004 2024-11-13T18:18:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741876_1052 (size=36767) 2024-11-13T18:18:56,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741876_1052 (size=36767) 2024-11-13T18:18:56,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741876_1052 (size=36767) 2024-11-13T18:18:56,881 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:18:56,883 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004 2024-11-13T18:18:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741875_1051 (size=36767) 2024-11-13T18:18:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741875_1051 (size=36767) 2024-11-13T18:18:56,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741875_1051 (size=36767) 2024-11-13T18:18:56,885 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:18:56,887 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004 2024-11-13T18:18:57,116 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:18:57,127 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004 2024-11-13T18:18:57,127 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 16 edits across 8 Regions in 732 ms; skipped=8; WAL=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395, size=290.7 K, length=297717, corrupted=false, cancelled=false 2024-11-13T18:18:57,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395, journal: Splitting hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395, size=290.7 K (297717bytes) at 1731521936360Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936465 (+105 ms)Finishing writing output for hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 so closing down at 1731521936481 (+16 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936489 (+8 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936493 (+4 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936500 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936507 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936509 (+2 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936537 (+28 ms)Creating recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp at 1731521936549 (+12 ms)3 split writer threads finished at 1731521936550 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731521936714 (+164 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004 at 1731521936718 (+4 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731521936719 (+1 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004 at 1731521936724 (+5 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 2 ms) at 1731521936801 (+77 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004 at 1731521936809 (+8 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731521936848 (+39 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731521936852 (+4 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004 at 1731521936858 (+6 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004 at 1731521936860 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731521936881 (+21 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004 at 1731521936883 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731521936885 (+2 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004 at 1731521936887 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731521937116 (+229 ms)Rename recovered edits hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004-fda96cf025dc%2C42475%2C1731521745166.1731521747395.temp to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004 at 1731521937127 (+11 ms)Processed 16 edits across 8 Regions in 732 ms; skipped=8; WAL=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395, size=290.7 K, length=297717, corrupted=false, cancelled=false at 1731521937127 2024-11-13T18:18:57,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 2024-11-13T18:18:57,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T18:18:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41233 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T18:18:57,136 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting/fda96cf025dc%2C42475%2C1731521745166.1731521747395 to hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs 2024-11-13T18:18:57,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T18:18:57,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 in 964 msec 2024-11-13T18:18:57,150 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-13T18:18:57,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 in 988 msec 2024-11-13T18:18:57,156 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting dir is empty, no logs to split. 2024-11-13T18:18:57,156 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=0, meta=false 2024-11-13T18:18:57,156 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,42475,1731521745166 WAL splitting is done? wals=0, meta=false 2024-11-13T18:18:57,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:18:57,158 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:57,159 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:57,403 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,403 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:57,403 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:57,403 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:57,403 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:57,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:57,404 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:57,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:57,404 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:57,405 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:57,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:57,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:57,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:57,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,410 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T18:18:57,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=61 2024-11-13T18:18:57,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,35435,1731521744991, table=hbase:meta, region=1588230740 2024-11-13T18:18:57,412 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:57,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,414 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,414 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521967416 2024-11-13T18:18:57,416 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,426 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:57,426 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521997426 2024-11-13T18:18:57,426 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522057426 2024-11-13T18:18:57,426 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-13T18:18:57,429 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server fda96cf025dc,42475,1731521745166 does not exist 2024-11-13T18:18:57,429 INFO [PEWorker-1 {}] master.SplitWALManager(105): fda96cf025dc,42475,1731521745166 WAL count=0, meta=false 2024-11-13T18:18:57,429 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,42475,1731521745166 WAL splitting is done? wals=0, meta=false 2024-11-13T18:18:57,430 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for fda96cf025dc,42475,1731521745166 failed, ignore...File hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,42475,1731521745166-splitting does not exist. 2024-11-13T18:18:57,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:57,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN}] 2024-11-13T18:18:57,432 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:18:57,432 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:57,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:57,676 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,677 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:57,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:57,677 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:57,677 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:57,677 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:57,677 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:57,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:57,678 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:57,678 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:57,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:57,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:57,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:57,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T18:18:57,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=61 2024-11-13T18:18:57,684 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,35435,1731521744991, table=hbase:meta, region=1588230740 2024-11-13T18:18:57,684 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:57,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:57,687 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,687 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,687 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:57,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521967692 2024-11-13T18:18:57,693 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,42475,1731521745166, regionLocation=fda96cf025dc,42475,1731521745166, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:57,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:57,700 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:57,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521997700 2024-11-13T18:18:57,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522057700 2024-11-13T18:18:57,700 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-13T18:18:57,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN}] 2024-11-13T18:18:57,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:57,702 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN 2024-11-13T18:18:57,703 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,704 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN 2024-11-13T18:18:57,705 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,706 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN 2024-11-13T18:18:57,707 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN 2024-11-13T18:18:57,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,709 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN 2024-11-13T18:18:57,710 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,711 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN 2024-11-13T18:18:57,711 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,712 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN 2024-11-13T18:18:57,713 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN 2024-11-13T18:18:57,714 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:18:57,854 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:18:57,854 INFO [fda96cf025dc:41233 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:18:57,854 DEBUG [fda96cf025dc:41233 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:18:57,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=fb720a25eb86c541f9c1f814843c651b, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN because future has completed 2024-11-13T18:18:57,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN because future has completed 2024-11-13T18:18:57,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:18:57,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN because future has completed 2024-11-13T18:18:57,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:18:57,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:18:57,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=02893942b5e22ded04197e4e8486e0dc, regionState=OPENING, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:57,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=8636f69311e6a46aa80948ea8be00961, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN because future has completed 2024-11-13T18:18:57,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=09ff4ab7a9d0091782b93c194b270c43, regionState=OPENING, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:57,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,46791,1731521745104}] 2024-11-13T18:18:57,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN because future has completed 2024-11-13T18:18:57,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN because future has completed 2024-11-13T18:18:57,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN because future has completed 2024-11-13T18:18:57,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN because future has completed 2024-11-13T18:18:57,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:18:57,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:18:57,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:18:57,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,35435,1731521744991}] 2024-11-13T18:18:58,019 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:18:58,019 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => fb720a25eb86c541f9c1f814843c651b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T18:18:58,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,020 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,022 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,023 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily11 2024-11-13T18:18:58,023 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,024 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,024 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,025 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily21 2024-11-13T18:18:58,025 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,026 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,026 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,026 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb720a25eb86c541f9c1f814843c651b columnFamilyName colfamily31 2024-11-13T18:18:58,026 DEBUG [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,027 INFO [StoreOpener-fb720a25eb86c541f9c1f814843c651b-1 {}] regionserver.HStore(327): Store=fb720a25eb86c541f9c1f814843c651b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,028 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,030 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,031 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,036 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:18:58,036 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 09ff4ab7a9d0091782b93c194b270c43, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T18:18:58,037 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,037 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,037 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,037 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,038 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,039 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,039 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(2902): Flushing fb720a25eb86c541f9c1f814843c651b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,040 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily11 2024-11-13T18:18:58,040 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,041 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,041 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,043 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily21 2024-11-13T18:18:58,043 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,044 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,044 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,045 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09ff4ab7a9d0091782b93c194b270c43 columnFamilyName colfamily31 2024-11-13T18:18:58,045 DEBUG [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,046 INFO [StoreOpener-09ff4ab7a9d0091782b93c194b270c43-1 {}] regionserver.HStore(327): Store=09ff4ab7a9d0091782b93c194b270c43/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,046 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,047 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,049 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,050 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004 2024-11-13T18:18:58,057 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004 2024-11-13T18:18:58,057 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing 09ff4ab7a9d0091782b93c194b270c43 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,073 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/.tmp/colfamily11/784f2f0439a44820af1629df988c6e7f is 52, key is uuu/colfamily11:colfamily11/1731521753514/Put/seqid=0 2024-11-13T18:18:58,079 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/.tmp/colfamily11/f89303555eec4409a9c5c991fa5230ab is 52, key is fff/colfamily11:colfamily11/1731521753268/Put/seqid=0 2024-11-13T18:18:58,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741877_1053 (size=39263) 2024-11-13T18:18:58,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741877_1053 (size=39263) 2024-11-13T18:18:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741877_1053 (size=39263) 2024-11-13T18:18:58,089 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/.tmp/colfamily11/784f2f0439a44820af1629df988c6e7f 2024-11-13T18:18:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741878_1054 (size=39263) 2024-11-13T18:18:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741878_1054 (size=39263) 2024-11-13T18:18:58,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741878_1054 (size=39263) 2024-11-13T18:18:58,091 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/.tmp/colfamily11/f89303555eec4409a9c5c991fa5230ab 2024-11-13T18:18:58,098 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/.tmp/colfamily11/784f2f0439a44820af1629df988c6e7f as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/colfamily11/784f2f0439a44820af1629df988c6e7f 2024-11-13T18:18:58,099 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/.tmp/colfamily11/f89303555eec4409a9c5c991fa5230ab as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/colfamily11/f89303555eec4409a9c5c991fa5230ab 2024-11-13T18:18:58,105 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/colfamily11/784f2f0439a44820af1629df988c6e7f, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,106 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb720a25eb86c541f9c1f814843c651b in 66ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,109 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,109 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/colfamily11/f89303555eec4409a9c5c991fa5230ab, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,109 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 09ff4ab7a9d0091782b93c194b270c43 in 52ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,110 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,110 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,111 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/0000000000000000004 2024-11-13T18:18:58,111 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,112 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,113 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,113 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,113 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,115 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,116 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,118 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened fb720a25eb86c541f9c1f814843c651b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66299398, jitterRate=-0.012061983346939087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,118 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:18:58,118 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for fb720a25eb86c541f9c1f814843c651b: Running coprocessor pre-open hook at 1731521938020Writing region info on filesystem at 1731521938020Initializing all the Stores at 1731521938021 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938021Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938022 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938022Obtaining lock to block concurrent updates at 1731521938039 (+17 ms)Preparing flush snapshotting stores in fb720a25eb86c541f9c1f814843c651b at 1731521938039Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938040 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. at 1731521938040Flushing fb720a25eb86c541f9c1f814843c651b/colfamily11: creating writer at 1731521938040Flushing fb720a25eb86c541f9c1f814843c651b/colfamily11: appending metadata at 1731521938073 (+33 ms)Flushing fb720a25eb86c541f9c1f814843c651b/colfamily11: closing flushed file at 1731521938073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a9b912: reopening flushed file at 1731521938097 (+24 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb720a25eb86c541f9c1f814843c651b in 66ms, sequenceid=4, compaction requested=false; wal=null at 1731521938106 (+9 ms)Cleaning up temporary data from old regions at 1731521938110 (+4 ms)Running coprocessor post-open hooks at 1731521938118 (+8 ms)Region opened successfully at 1731521938118 2024-11-13T18:18:58,118 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,119 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., pid=72, masterSystemTime=1731521938014 2024-11-13T18:18:58,120 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 09ff4ab7a9d0091782b93c194b270c43; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67901702, jitterRate=0.011814206838607788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,120 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:18:58,120 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 09ff4ab7a9d0091782b93c194b270c43: Running coprocessor pre-open hook at 1731521938037Writing region info on filesystem at 1731521938037Initializing all the Stores at 1731521938038 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938038Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938038Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938038Obtaining lock to block concurrent updates at 1731521938057 (+19 ms)Preparing flush snapshotting stores in 09ff4ab7a9d0091782b93c194b270c43 at 1731521938057Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938057Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. at 1731521938057Flushing 09ff4ab7a9d0091782b93c194b270c43/colfamily11: creating writer at 1731521938057Flushing 09ff4ab7a9d0091782b93c194b270c43/colfamily11: appending metadata at 1731521938079 (+22 ms)Flushing 09ff4ab7a9d0091782b93c194b270c43/colfamily11: closing flushed file at 1731521938079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b392924: reopening flushed file at 1731521938098 (+19 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 09ff4ab7a9d0091782b93c194b270c43 in 52ms, sequenceid=4, compaction requested=false; wal=null at 1731521938109 (+11 ms)Cleaning up temporary data from old regions at 1731521938113 (+4 ms)Running coprocessor post-open hooks at 1731521938120 (+7 ms)Region opened successfully at 1731521938120 2024-11-13T18:18:58,121 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., pid=76, masterSystemTime=1731521938033 2024-11-13T18:18:58,122 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:18:58,122 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:18:58,123 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:18:58,123 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:58,123 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 02893942b5e22ded04197e4e8486e0dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-13T18:18:58,123 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,123 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,123 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,123 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,124 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:18:58,124 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:18:58,124 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:18:58,124 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 8636f69311e6a46aa80948ea8be00961, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-13T18:18:58,124 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,125 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,125 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,125 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:18:58,125 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,127 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:58,127 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily11 2024-11-13T18:18:58,127 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,128 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,128 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily11 2024-11-13T18:18:58,128 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,128 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,129 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,129 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,129 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily21 2024-11-13T18:18:58,129 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,130 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily21 2024-11-13T18:18:58,130 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:18:58,130 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,131 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,131 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,131 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,131 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8636f69311e6a46aa80948ea8be00961 columnFamilyName colfamily31 2024-11-13T18:18:58,132 DEBUG [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,132 INFO [StoreOpener-8636f69311e6a46aa80948ea8be00961-1 {}] regionserver.HStore(327): Store=8636f69311e6a46aa80948ea8be00961/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,132 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,133 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=68 2024-11-13T18:18:58,135 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02893942b5e22ded04197e4e8486e0dc columnFamilyName colfamily31 2024-11-13T18:18:58,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,46791,1731521745104 in 269 msec 2024-11-13T18:18:58,135 DEBUG [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,135 INFO [StoreOpener-02893942b5e22ded04197e4e8486e0dc-1 {}] regionserver.HStore(327): Store=02893942b5e22ded04197e4e8486e0dc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,135 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,136 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN in 435 msec 2024-11-13T18:18:58,137 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,137 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004 2024-11-13T18:18:58,139 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=67 2024-11-13T18:18:58,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,35435,1731521744991 in 257 msec 2024-11-13T18:18:58,141 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004 2024-11-13T18:18:58,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN in 440 msec 2024-11-13T18:18:58,145 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004 2024-11-13T18:18:58,145 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 8636f69311e6a46aa80948ea8be00961 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,147 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004 2024-11-13T18:18:58,147 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 02893942b5e22ded04197e4e8486e0dc 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,185 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/.tmp/colfamily11/fd3fd71fd25e42e3b92d89d3626ce2f9 is 52, key is lll/colfamily11:colfamily11/1731521753367/Put/seqid=0 2024-11-13T18:18:58,188 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/.tmp/colfamily11/fbb7d5dc2620436cb7dad05fd6e1ceb5 is 52, key is ccc/colfamily11:colfamily11/1731521753220/Put/seqid=0 2024-11-13T18:18:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741879_1055 (size=39263) 2024-11-13T18:18:58,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741879_1055 (size=39263) 2024-11-13T18:18:58,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741879_1055 (size=39263) 2024-11-13T18:18:58,248 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/.tmp/colfamily11/fbb7d5dc2620436cb7dad05fd6e1ceb5 2024-11-13T18:18:58,259 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/.tmp/colfamily11/fbb7d5dc2620436cb7dad05fd6e1ceb5 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/colfamily11/fbb7d5dc2620436cb7dad05fd6e1ceb5 2024-11-13T18:18:58,270 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/colfamily11/fbb7d5dc2620436cb7dad05fd6e1ceb5, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,271 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 02893942b5e22ded04197e4e8486e0dc in 123ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,275 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/0000000000000000004 2024-11-13T18:18:58,279 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,279 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741880_1056 (size=39263) 2024-11-13T18:18:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741880_1056 (size=39263) 2024-11-13T18:18:58,280 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741880_1056 (size=39263) 2024-11-13T18:18:58,282 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/.tmp/colfamily11/fd3fd71fd25e42e3b92d89d3626ce2f9 2024-11-13T18:18:58,285 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,290 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,298 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 02893942b5e22ded04197e4e8486e0dc; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68608732, jitterRate=0.022349774837493896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,298 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:18:58,298 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 02893942b5e22ded04197e4e8486e0dc: Running coprocessor pre-open hook at 1731521938123Writing region info on filesystem at 1731521938124 (+1 ms)Initializing all the Stores at 1731521938125 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938125Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938125Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938125Obtaining lock to block concurrent updates at 1731521938147 (+22 ms)Preparing flush snapshotting stores in 02893942b5e22ded04197e4e8486e0dc at 1731521938147Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938147Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. at 1731521938147Flushing 02893942b5e22ded04197e4e8486e0dc/colfamily11: creating writer at 1731521938147Flushing 02893942b5e22ded04197e4e8486e0dc/colfamily11: appending metadata at 1731521938187 (+40 ms)Flushing 02893942b5e22ded04197e4e8486e0dc/colfamily11: closing flushed file at 1731521938187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79169d37: reopening flushed file at 1731521938257 (+70 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 02893942b5e22ded04197e4e8486e0dc in 123ms, sequenceid=4, compaction requested=false; wal=null at 1731521938271 (+14 ms)Cleaning up temporary data from old regions at 1731521938279 (+8 ms)Running coprocessor post-open hooks at 1731521938298 (+19 ms)Region opened successfully at 1731521938298 2024-11-13T18:18:58,300 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., pid=75, masterSystemTime=1731521938014 2024-11-13T18:18:58,300 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/.tmp/colfamily11/fd3fd71fd25e42e3b92d89d3626ce2f9 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/colfamily11/fd3fd71fd25e42e3b92d89d3626ce2f9 2024-11-13T18:18:58,305 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:18:58,305 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:18:58,305 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:18:58,305 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eefcb355d011c4ea65b0572629d5ef5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T18:18:58,306 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,306 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:58,306 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,306 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,306 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:18:58,316 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/colfamily11/fd3fd71fd25e42e3b92d89d3626ce2f9, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,316 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8636f69311e6a46aa80948ea8be00961 in 171ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,320 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/0000000000000000004 2024-11-13T18:18:58,321 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,324 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,325 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,325 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily11 2024-11-13T18:18:58,325 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=69 2024-11-13T18:18:58,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,46791,1731521745104 in 449 msec 2024-11-13T18:18:58,328 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN in 626 msec 2024-11-13T18:18:58,334 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,335 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,338 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily21 2024-11-13T18:18:58,338 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,338 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,339 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,339 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,340 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eefcb355d011c4ea65b0572629d5ef5 columnFamilyName colfamily31 2024-11-13T18:18:58,340 DEBUG [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,341 INFO [StoreOpener-3eefcb355d011c4ea65b0572629d5ef5-1 {}] regionserver.HStore(327): Store=3eefcb355d011c4ea65b0572629d5ef5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,341 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,343 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,344 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,345 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 8636f69311e6a46aa80948ea8be00961; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72778435, jitterRate=0.08448319137096405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,345 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:18:58,346 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 8636f69311e6a46aa80948ea8be00961: Running coprocessor pre-open hook at 1731521938125Writing region info on filesystem at 1731521938125Initializing all the Stores at 1731521938126 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938126Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938126Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938127 (+1 ms)Obtaining lock to block concurrent updates at 1731521938145 (+18 ms)Preparing flush snapshotting stores in 8636f69311e6a46aa80948ea8be00961 at 1731521938145Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938145Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. at 1731521938145Flushing 8636f69311e6a46aa80948ea8be00961/colfamily11: creating writer at 1731521938145Flushing 8636f69311e6a46aa80948ea8be00961/colfamily11: appending metadata at 1731521938184 (+39 ms)Flushing 8636f69311e6a46aa80948ea8be00961/colfamily11: closing flushed file at 1731521938184Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@485a7290: reopening flushed file at 1731521938294 (+110 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8636f69311e6a46aa80948ea8be00961 in 171ms, sequenceid=4, compaction requested=false; wal=null at 1731521938316 (+22 ms)Cleaning up temporary data from old regions at 1731521938325 (+9 ms)Running coprocessor post-open hooks at 1731521938345 (+20 ms)Region opened successfully at 1731521938345 2024-11-13T18:18:58,347 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., pid=77, masterSystemTime=1731521938033 2024-11-13T18:18:58,347 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,349 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004 2024-11-13T18:18:58,352 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:18:58,352 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:18:58,352 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:18:58,353 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 400b285d3ec5b0d43bcd37cdfa6b8f90, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-13T18:18:58,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:58,353 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,353 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,353 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,353 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:18:58,358 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004 2024-11-13T18:18:58,358 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 3eefcb355d011c4ea65b0572629d5ef5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,360 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=71 2024-11-13T18:18:58,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,35435,1731521744991 in 475 msec 2024-11-13T18:18:58,364 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily11 2024-11-13T18:18:58,364 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,365 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN in 664 msec 2024-11-13T18:18:58,366 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,369 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily21 2024-11-13T18:18:58,371 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,374 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,374 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,376 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400b285d3ec5b0d43bcd37cdfa6b8f90 columnFamilyName colfamily31 2024-11-13T18:18:58,376 DEBUG [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,376 INFO [StoreOpener-400b285d3ec5b0d43bcd37cdfa6b8f90-1 {}] regionserver.HStore(327): Store=400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,376 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,378 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,381 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,383 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004 2024-11-13T18:18:58,390 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004 2024-11-13T18:18:58,390 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 400b285d3ec5b0d43bcd37cdfa6b8f90 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,396 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/.tmp/colfamily11/6fd973a0b8bb4eb3b539727de0cd2d78 is 52, key is ooo/colfamily11:colfamily11/1731521753412/Put/seqid=0 2024-11-13T18:18:58,416 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/.tmp/colfamily11/4a72dbe1741c49c5bdabd8f8628ae6e3 is 52, key is xxx/colfamily11:colfamily11/1731521753585/Put/seqid=0 2024-11-13T18:18:58,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741881_1057 (size=39263) 2024-11-13T18:18:58,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741881_1057 (size=39263) 2024-11-13T18:18:58,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741881_1057 (size=39263) 2024-11-13T18:18:58,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741882_1058 (size=39263) 2024-11-13T18:18:58,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741882_1058 (size=39263) 2024-11-13T18:18:58,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741882_1058 (size=39263) 2024-11-13T18:18:58,509 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/.tmp/colfamily11/4a72dbe1741c49c5bdabd8f8628ae6e3 2024-11-13T18:18:58,535 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/.tmp/colfamily11/4a72dbe1741c49c5bdabd8f8628ae6e3 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11/4a72dbe1741c49c5bdabd8f8628ae6e3 2024-11-13T18:18:58,551 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11/4a72dbe1741c49c5bdabd8f8628ae6e3, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,551 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 400b285d3ec5b0d43bcd37cdfa6b8f90 in 161ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,553 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/0000000000000000004 2024-11-13T18:18:58,555 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,555 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,556 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,563 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,569 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,571 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 400b285d3ec5b0d43bcd37cdfa6b8f90; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64616179, jitterRate=-0.037143900990486145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,572 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:18:58,572 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 400b285d3ec5b0d43bcd37cdfa6b8f90: Running coprocessor pre-open hook at 1731521938354Writing region info on filesystem at 1731521938354Initializing all the Stores at 1731521938355 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938355Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938357 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938357Obtaining lock to block concurrent updates at 1731521938390 (+33 ms)Preparing flush snapshotting stores in 400b285d3ec5b0d43bcd37cdfa6b8f90 at 1731521938390Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938391 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. at 1731521938391Flushing 400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11: creating writer at 1731521938391Flushing 400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11: appending metadata at 1731521938416 (+25 ms)Flushing 400b285d3ec5b0d43bcd37cdfa6b8f90/colfamily11: closing flushed file at 1731521938416Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20b627aa: reopening flushed file at 1731521938534 (+118 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 400b285d3ec5b0d43bcd37cdfa6b8f90 in 161ms, sequenceid=4, compaction requested=false; wal=null at 1731521938551 (+17 ms)Cleaning up temporary data from old regions at 1731521938555 (+4 ms)Running coprocessor post-open hooks at 1731521938572 (+17 ms)Region opened successfully at 1731521938572 2024-11-13T18:18:58,573 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., pid=79, masterSystemTime=1731521938033 2024-11-13T18:18:58,581 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:18:58,581 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:18:58,581 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:18:58,582 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 64b5086eb7020ccba10769de2eaf3d9b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-13T18:18:58,582 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,582 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,582 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,582 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,586 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,588 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily11 2024-11-13T18:18:58,588 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:58,590 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,590 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,591 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily21 2024-11-13T18:18:58,591 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:18:58,592 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,592 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,593 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5086eb7020ccba10769de2eaf3d9b columnFamilyName colfamily31 2024-11-13T18:18:58,593 DEBUG [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,594 INFO [StoreOpener-64b5086eb7020ccba10769de2eaf3d9b-1 {}] regionserver.HStore(327): Store=64b5086eb7020ccba10769de2eaf3d9b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,595 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,598 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=66 2024-11-13T18:18:58,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,35435,1731521744991 in 707 msec 2024-11-13T18:18:58,601 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,602 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN in 901 msec 2024-11-13T18:18:58,608 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,608 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 64b5086eb7020ccba10769de2eaf3d9b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:58,630 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/.tmp/colfamily11/2ec4576151d34d2b9a230892fc3f721a is 52, key is rrr/colfamily11:colfamily11/1731521753464/Put/seqid=0 2024-11-13T18:18:58,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741883_1059 (size=39263) 2024-11-13T18:18:58,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741883_1059 (size=39263) 2024-11-13T18:18:58,692 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/.tmp/colfamily11/2ec4576151d34d2b9a230892fc3f721a 2024-11-13T18:18:58,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741883_1059 (size=39263) 2024-11-13T18:18:58,716 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/.tmp/colfamily11/2ec4576151d34d2b9a230892fc3f721a as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/colfamily11/2ec4576151d34d2b9a230892fc3f721a 2024-11-13T18:18:58,727 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/colfamily11/2ec4576151d34d2b9a230892fc3f721a, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,728 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 64b5086eb7020ccba10769de2eaf3d9b in 119ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,729 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/0000000000000000004 2024-11-13T18:18:58,732 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,732 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,734 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,736 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,745 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,747 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 64b5086eb7020ccba10769de2eaf3d9b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64336731, jitterRate=-0.041308000683784485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,747 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:18:58,747 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 64b5086eb7020ccba10769de2eaf3d9b: Running coprocessor pre-open hook at 1731521938582Writing region info on filesystem at 1731521938582Initializing all the Stores at 1731521938586 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938586Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938586Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938586Obtaining lock to block concurrent updates at 1731521938608 (+22 ms)Preparing flush snapshotting stores in 64b5086eb7020ccba10769de2eaf3d9b at 1731521938608Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938608Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. at 1731521938608Flushing 64b5086eb7020ccba10769de2eaf3d9b/colfamily11: creating writer at 1731521938608Flushing 64b5086eb7020ccba10769de2eaf3d9b/colfamily11: appending metadata at 1731521938630 (+22 ms)Flushing 64b5086eb7020ccba10769de2eaf3d9b/colfamily11: closing flushed file at 1731521938630Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ada0558: reopening flushed file at 1731521938715 (+85 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 64b5086eb7020ccba10769de2eaf3d9b in 119ms, sequenceid=4, compaction requested=false; wal=null at 1731521938728 (+13 ms)Cleaning up temporary data from old regions at 1731521938732 (+4 ms)Running coprocessor post-open hooks at 1731521938747 (+15 ms)Region opened successfully at 1731521938747 2024-11-13T18:18:58,751 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., pid=78, masterSystemTime=1731521938033 2024-11-13T18:18:58,765 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:18:58,765 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:18:58,767 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:58,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,35435,1731521744991 because future has completed 2024-11-13T18:18:58,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=65 2024-11-13T18:18:58,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,35435,1731521744991 in 891 msec 2024-11-13T18:18:58,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN in 1.0810 sec 2024-11-13T18:18:58,897 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/.tmp/colfamily11/6fd973a0b8bb4eb3b539727de0cd2d78 2024-11-13T18:18:58,928 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/.tmp/colfamily11/6fd973a0b8bb4eb3b539727de0cd2d78 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/colfamily11/6fd973a0b8bb4eb3b539727de0cd2d78 2024-11-13T18:18:58,939 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/colfamily11/6fd973a0b8bb4eb3b539727de0cd2d78, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:58,940 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3eefcb355d011c4ea65b0572629d5ef5 in 581ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:58,941 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/0000000000000000004 2024-11-13T18:18:58,943 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,943 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,944 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:58,948 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,953 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:58,956 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 3eefcb355d011c4ea65b0572629d5ef5; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69184571, jitterRate=0.030930444598197937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:58,957 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:18:58,957 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 3eefcb355d011c4ea65b0572629d5ef5: Running coprocessor pre-open hook at 1731521938306Writing region info on filesystem at 1731521938306Initializing all the Stores at 1731521938319 (+13 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938319Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938321 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938321Obtaining lock to block concurrent updates at 1731521938358 (+37 ms)Preparing flush snapshotting stores in 3eefcb355d011c4ea65b0572629d5ef5 at 1731521938358Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521938358Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. at 1731521938358Flushing 3eefcb355d011c4ea65b0572629d5ef5/colfamily11: creating writer at 1731521938359 (+1 ms)Flushing 3eefcb355d011c4ea65b0572629d5ef5/colfamily11: appending metadata at 1731521938396 (+37 ms)Flushing 3eefcb355d011c4ea65b0572629d5ef5/colfamily11: closing flushed file at 1731521938396Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4300d8a8: reopening flushed file at 1731521938924 (+528 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3eefcb355d011c4ea65b0572629d5ef5 in 581ms, sequenceid=4, compaction requested=false; wal=null at 1731521938940 (+16 ms)Cleaning up temporary data from old regions at 1731521938943 (+3 ms)Running coprocessor post-open hooks at 1731521938957 (+14 ms)Region opened successfully at 1731521938957 2024-11-13T18:18:58,958 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., pid=74, masterSystemTime=1731521938014 2024-11-13T18:18:58,962 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:18:58,962 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:18:58,963 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:18:58,963 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fb2e398d735ceeb281622c9b30d4cf1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T18:18:58,963 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,963 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:18:58,963 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,964 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:58,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:18:58,972 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,975 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily11 2024-11-13T18:18:58,975 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,976 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,976 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,978 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily21 2024-11-13T18:18:58,978 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,981 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,981 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=70 2024-11-13T18:18:58,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,46791,1731521745104 in 1.1060 sec 2024-11-13T18:18:58,985 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb2e398d735ceeb281622c9b30d4cf1 columnFamilyName colfamily31 2024-11-13T18:18:58,985 DEBUG [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:18:58,986 INFO [StoreOpener-7fb2e398d735ceeb281622c9b30d4cf1-1 {}] regionserver.HStore(327): Store=7fb2e398d735ceeb281622c9b30d4cf1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:18:58,986 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,987 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN in 1.2850 sec 2024-11-13T18:18:58,990 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:58,993 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004 2024-11-13T18:18:59,003 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004 2024-11-13T18:18:59,004 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 7fb2e398d735ceeb281622c9b30d4cf1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:18:59,054 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/.tmp/colfamily11/6ee8c9a82caa497f8ef43b5c4cf46b1b is 52, key is iii/colfamily11:colfamily11/1731521753316/Put/seqid=0 2024-11-13T18:18:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741884_1060 (size=39263) 2024-11-13T18:18:59,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741884_1060 (size=39263) 2024-11-13T18:18:59,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741884_1060 (size=39263) 2024-11-13T18:18:59,502 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/.tmp/colfamily11/6ee8c9a82caa497f8ef43b5c4cf46b1b 2024-11-13T18:18:59,511 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/.tmp/colfamily11/6ee8c9a82caa497f8ef43b5c4cf46b1b as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/colfamily11/6ee8c9a82caa497f8ef43b5c4cf46b1b 2024-11-13T18:18:59,523 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/colfamily11/6ee8c9a82caa497f8ef43b5c4cf46b1b, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T18:18:59,524 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7fb2e398d735ceeb281622c9b30d4cf1 in 520ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T18:18:59,525 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/0000000000000000004 2024-11-13T18:18:59,529 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:59,529 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:59,530 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:18:59,532 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:59,544 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:18:59,545 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 7fb2e398d735ceeb281622c9b30d4cf1; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65081324, jitterRate=-0.030212700366973877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:18:59,546 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:18:59,546 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 7fb2e398d735ceeb281622c9b30d4cf1: Running coprocessor pre-open hook at 1731521938964Writing region info on filesystem at 1731521938964Initializing all the Stores at 1731521938966 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938966Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938969 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731521938969Obtaining lock to block concurrent updates at 1731521939004 (+35 ms)Preparing flush snapshotting stores in 7fb2e398d735ceeb281622c9b30d4cf1 at 1731521939004Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521939004Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. at 1731521939004Flushing 7fb2e398d735ceeb281622c9b30d4cf1/colfamily11: creating writer at 1731521939004Flushing 7fb2e398d735ceeb281622c9b30d4cf1/colfamily11: appending metadata at 1731521939053 (+49 ms)Flushing 7fb2e398d735ceeb281622c9b30d4cf1/colfamily11: closing flushed file at 1731521939053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8fca966: reopening flushed file at 1731521939509 (+456 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7fb2e398d735ceeb281622c9b30d4cf1 in 520ms, sequenceid=4, compaction requested=false; wal=null at 1731521939524 (+15 ms)Cleaning up temporary data from old regions at 1731521939529 (+5 ms)Running coprocessor post-open hooks at 1731521939546 (+17 ms)Region opened successfully at 1731521939546 2024-11-13T18:18:59,547 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., pid=73, masterSystemTime=1731521938014 2024-11-13T18:18:59,551 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:18:59,551 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:18:59,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, openSeqNum=5, regionLocation=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,46791,1731521745104 because future has completed 2024-11-13T18:18:59,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=64 2024-11-13T18:18:59,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,46791,1731521745104 in 1.6960 sec 2024-11-13T18:18:59,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=57 2024-11-13T18:18:59,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN in 1.8640 sec 2024-11-13T18:18:59,567 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server fda96cf025dc,42475,1731521745166 after splitting done 2024-11-13T18:18:59,567 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T18:18:59,567 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:59,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:59,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:18:59,701 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:59,701 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T18:18:59,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:18:59,701 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:18:59,701 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:18:59,702 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T18:18:59,702 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T18:18:59,702 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T18:18:59,702 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:18:59,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T18:18:59,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:18:59,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN 2024-11-13T18:18:59,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,46791,1731521745104 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,35435,1731521744991 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.meta.1731521747890.meta, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C42475%2C1731521745166.1731521747395, worker=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7fb2e398d735ceeb281622c9b30d4cf1, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64b5086eb7020ccba10769de2eaf3d9b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=400b285d3ec5b0d43bcd37cdfa6b8f90, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=09ff4ab7a9d0091782b93c194b270c43, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb720a25eb86c541f9c1f814843c651b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=02893942b5e22ded04197e4e8486e0dc, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3eefcb355d011c4ea65b0572629d5ef5, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8636f69311e6a46aa80948ea8be00961, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure fb720a25eb86c541f9c1f814843c651b, server=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fb2e398d735ceeb281622c9b30d4cf1, server=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eefcb355d011c4ea65b0572629d5ef5, server=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 02893942b5e22ded04197e4e8486e0dc, server=fda96cf025dc,46791,1731521745104 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 09ff4ab7a9d0091782b93c194b270c43, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 8636f69311e6a46aa80948ea8be00961, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 64b5086eb7020ccba10769de2eaf3d9b, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,712 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 400b285d3ec5b0d43bcd37cdfa6b8f90, server=fda96cf025dc,35435,1731521744991 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T18:18:59,712 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-13T18:18:59,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=61 2024-11-13T18:18:59,713 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fda96cf025dc,35435,1731521744991, table=hbase:meta, region=1588230740 2024-11-13T18:18:59,713 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,35435,1731521744991, state=OPEN 2024-11-13T18:18:59,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:59,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:59,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:18:59,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:59,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:59,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:18:59,716 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731521969716 2024-11-13T18:18:59,716 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd437c88e4af73574ce0ce0882aca142, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38c892b7353daa24a9b6c917f1aa0a89, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02893942b5e22ded04197e4e8486e0dc, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=5 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00a4d576f4c41497f18224865b73403a, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e715c3850b0001604b62543d4e17908, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=09ff4ab7a9d0091782b93c194b270c43, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=5 2024-11-13T18:18:59,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=801d0f207e45efea50135a16f5fcbc99, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c39d6007aedf97e4c4a94511fe061ef, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb2e398d735ceeb281622c9b30d4cf1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=5 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5aa68e8ac635bc9e7d6868f5f771281e, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e5bab85a9454bcf58ade3574335c3bd1, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8636f69311e6a46aa80948ea8be00961, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=5 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27ae2919589ed9f92a8ad04abb8e24d5, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37863ce38db650b86b62a2bccc5e7796, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3eefcb355d011c4ea65b0572629d5ef5, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=5 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=575404eded2e7a15cbfa9186eea1d1b1, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dae4cf552a1afaf0b041668499d99e20, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5086eb7020ccba10769de2eaf3d9b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=5 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b91248305f9bd170badfc1ac16c2f2b, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=644025ac5ad44a7ca4a26c89d6611d8b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb720a25eb86c541f9c1f814843c651b, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=5 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a59239e975bc7ad59b0ae9308416d51, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b196a2f44f4f468f8bb9a2e59b301747, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400b285d3ec5b0d43bcd37cdfa6b8f90, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=5 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=272983331631897f6cfeac3ce6bd7466, regionState=OPEN, lastHost=fda96cf025dc,35435,1731521744991, regionLocation=fda96cf025dc,35435,1731521744991, openSeqNum=2 2024-11-13T18:18:59,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf5aa9737456e8265a9b1e93d818d677, regionState=OPEN, lastHost=fda96cf025dc,46791,1731521745104, regionLocation=fda96cf025dc,46791,1731521745104, openSeqNum=2 2024-11-13T18:18:59,726 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T18:18:59,726 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731521999726 2024-11-13T18:18:59,726 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522059726 2024-11-13T18:18:59,727 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-13T18:18:59,727 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server fda96cf025dc,42475,1731521745166 after splitting done 2024-11-13T18:18:59,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T18:18:59,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure fda96cf025dc,42475,1731521745166, splitWal=true, meta=true in 3 mins, 5.252 sec 2024-11-13T18:18:59,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:18:59,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:18:59,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:18:59,996 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:42475 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:42475 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:18:59,997 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=java.net.ConnectException: Call to address=fda96cf025dc:42475 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:18:59,997 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-13T18:18:59,997 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:18:59,998 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:42475 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:42475 2024-11-13T18:19:00,107 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,42475,1731521745166, seqNum=-1 from cache 2024-11-13T18:19:00,216 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:19:00,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:19:00,219 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,35435,1731521744991, seqNum=-1] 2024-11-13T18:19:00,229 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc., hostname=fda96cf025dc,46791,1731521745104, seqNum=5] 2024-11-13T18:19:00,254 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,255 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,255 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,255 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:00,364 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43., hostname=fda96cf025dc,35435,1731521744991, seqNum=5] 2024-11-13T18:19:00,392 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,393 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,393 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,393 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:00,505 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1., hostname=fda96cf025dc,46791,1731521745104, seqNum=5] 2024-11-13T18:19:00,534 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,535 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,535 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,535 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:00,537 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:19:00,652 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961., hostname=fda96cf025dc,35435,1731521744991, seqNum=5] 2024-11-13T18:19:00,682 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:00,792 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5., hostname=fda96cf025dc,46791,1731521745104, seqNum=5] 2024-11-13T18:19:00,811 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,812 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,812 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,812 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:00,922 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b., hostname=fda96cf025dc,35435,1731521744991, seqNum=5] 2024-11-13T18:19:00,939 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:00,940 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,940 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:00,940 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:01,050 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b., hostname=fda96cf025dc,46791,1731521745104, seqNum=5] 2024-11-13T18:19:01,064 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:42475 this server is in the failed servers list 2024-11-13T18:19:01,065 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:42475 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:01,065 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:42475 2024-11-13T18:19:01,065 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,42475,1731521745166, seqNum=2 from cache 2024-11-13T18:19:01,185 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90., hostname=fda96cf025dc,35435,1731521744991, seqNum=5] 2024-11-13T18:19:01,369 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T18:19:01,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:19:01,370 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:19:01,370 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:19:01,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:01,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:01,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:19:01,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:19:01,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=534022733, stopped=false 2024-11-13T18:19:01,371 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda96cf025dc,41233,1731521744053 2024-11-13T18:19:01,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:01,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:01,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:19:01,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:19:01,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:19:01,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:01,374 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,35435,1731521744991' ***** 2024-11-13T18:19:01,374 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:19:01,374 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,46791,1731521745104' ***** 2024-11-13T18:19:01,374 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:19:01,375 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:19:01,375 INFO [RS:1;fda96cf025dc:46791 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:19:01,375 INFO [RS:1;fda96cf025dc:46791 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:19:01,376 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 7fb2e398d735ceeb281622c9b30d4cf1 2024-11-13T18:19:01,376 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:19:01,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:01,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:01,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:01,380 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:19:01,380 INFO [RS:0;fda96cf025dc:35435 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:19:01,380 INFO [RS:0;fda96cf025dc:35435 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:19:01,380 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e715c3850b0001604b62543d4e17908 2024-11-13T18:19:01,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:19:01,380 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 8636f69311e6a46aa80948ea8be00961 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 5aa68e8ac635bc9e7d6868f5f771281e 2024-11-13T18:19:01,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 02893942b5e22ded04197e4e8486e0dc 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 3eefcb355d011c4ea65b0572629d5ef5 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 5c39d6007aedf97e4c4a94511fe061ef 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 575404eded2e7a15cbfa9186eea1d1b1 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for fb720a25eb86c541f9c1f814843c651b 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 37863ce38db650b86b62a2bccc5e7796 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 644025ac5ad44a7ca4a26c89d6611d8b 2024-11-13T18:19:01,381 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a59239e975bc7ad59b0ae9308416d51 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 272983331631897f6cfeac3ce6bd7466 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 09ff4ab7a9d0091782b93c194b270c43 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,46791,1731521745104 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for b196a2f44f4f468f8bb9a2e59b301747 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 00a4d576f4c41497f18224865b73403a 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 64b5086eb7020ccba10769de2eaf3d9b 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 27ae2919589ed9f92a8ad04abb8e24d5 2024-11-13T18:19:01,381 INFO [RS:1;fda96cf025dc:46791 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fda96cf025dc:46791. 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 400b285d3ec5b0d43bcd37cdfa6b8f90 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:19:01,381 DEBUG [RS:1;fda96cf025dc:46791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:19:01,381 DEBUG [RS:1;fda96cf025dc:46791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b91248305f9bd170badfc1ac16c2f2b 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:19:01,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,35435,1731521744991 2024-11-13T18:19:01,381 INFO [RS:0;fda96cf025dc:35435 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:19:01,382 INFO [RS:0;fda96cf025dc:35435 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda96cf025dc:35435. 2024-11-13T18:19:01,382 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-13T18:19:01,382 DEBUG [RS:0;fda96cf025dc:35435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:19:01,382 DEBUG [RS:0;fda96cf025dc:35435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:01,382 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:19:01,382 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:19:01,382 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:19:01,382 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 02893942b5e22ded04197e4e8486e0dc, 1e715c3850b0001604b62543d4e17908, 37863ce38db650b86b62a2bccc5e7796, 3eefcb355d011c4ea65b0572629d5ef5, 575404eded2e7a15cbfa9186eea1d1b1, 5a59239e975bc7ad59b0ae9308416d51, 5aa68e8ac635bc9e7d6868f5f771281e, 5c39d6007aedf97e4c4a94511fe061ef, 644025ac5ad44a7ca4a26c89d6611d8b, 7fb2e398d735ceeb281622c9b30d4cf1, cf5aa9737456e8265a9b1e93d818d677, fb720a25eb86c541f9c1f814843c651b, fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:01,382 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8636f69311e6a46aa80948ea8be00961, disabling compactions & flushes 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7fb2e398d735ceeb281622c9b30d4cf1, disabling compactions & flushes 2024-11-13T18:19:01,383 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:19:01,383 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. after waiting 0 ms 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. after waiting 0 ms 2024-11-13T18:19:01,383 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:19:01,384 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-13T18:19:01,384 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 09ff4ab7a9d0091782b93c194b270c43, 1588230740, 1b91248305f9bd170badfc1ac16c2f2b, 272983331631897f6cfeac3ce6bd7466, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, 8636f69311e6a46aa80948ea8be00961, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:01,384 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:19:01,384 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:19:01,384 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:19:01,384 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:19:01,384 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:19:01,385 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=11.91 KB heapSize=17.63 KB 2024-11-13T18:19:01,389 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:19:01,423 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/d9ddf2ec8e0e44c3a1d80e0e6869cb5f is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc./info:regioninfo/1731521938306/Put/seqid=0 2024-11-13T18:19:01,436 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7fb2e398d735ceeb281622c9b30d4cf1/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:01,448 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:19:01,449 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:19:01,449 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8636f69311e6a46aa80948ea8be00961/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:01,461 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:19:01,461 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7fb2e398d735ceeb281622c9b30d4cf1: Waiting for close lock at 1731521941383Running coprocessor pre-close hooks at 1731521941383Disabling compacts and flushes for region at 1731521941383Disabling writes for close at 1731521941383Writing region close event to WAL at 1731521941404 (+21 ms)Running coprocessor post-close hooks at 1731521941453 (+49 ms)Closed at 1731521941461 (+8 ms) 2024-11-13T18:19:01,462 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731521748491.7fb2e398d735ceeb281622c9b30d4cf1. 2024-11-13T18:19:01,462 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e715c3850b0001604b62543d4e17908, disabling compactions & flushes 2024-11-13T18:19:01,462 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:19:01,462 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:19:01,462 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. after waiting 0 ms 2024-11-13T18:19:01,462 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:19:01,462 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1e715c3850b0001604b62543d4e17908 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:01,463 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8636f69311e6a46aa80948ea8be00961: Waiting for close lock at 1731521941383Running coprocessor pre-close hooks at 1731521941383Disabling compacts and flushes for region at 1731521941383Disabling writes for close at 1731521941383Writing region close event to WAL at 1731521941405 (+22 ms)Running coprocessor post-close hooks at 1731521941454 (+49 ms)Closed at 1731521941463 (+9 ms) 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731521748491.8636f69311e6a46aa80948ea8be00961. 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 272983331631897f6cfeac3ce6bd7466, disabling compactions & flushes 2024-11-13T18:19:01,463 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. after waiting 0 ms 2024-11-13T18:19:01,463 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:19:01,464 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 272983331631897f6cfeac3ce6bd7466 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:01,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741885_1061 (size=18322) 2024-11-13T18:19:01,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741885_1061 (size=18322) 2024-11-13T18:19:01,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741885_1061 (size=18322) 2024-11-13T18:19:01,495 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/d9ddf2ec8e0e44c3a1d80e0e6869cb5f 2024-11-13T18:19:01,507 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/.tmp/info/d9ddf2ec8e0e44c3a1d80e0e6869cb5f as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/info/d9ddf2ec8e0e44c3a1d80e0e6869cb5f 2024-11-13T18:19:01,510 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/.tmp/colfamily11/add785c54a0049738830713c39642d34 is 52, key is eee/colfamily11:colfamily11/1731521752955/Put/seqid=0 2024-11-13T18:19:01,523 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/info/d9ddf2ec8e0e44c3a1d80e0e6869cb5f, entries=64, sequenceid=79, filesize=17.9 K 2024-11-13T18:19:01,525 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 141ms, sequenceid=79, compaction requested=false 2024-11-13T18:19:01,526 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/.tmp/colfamily11/c44c426d3b7241149f8f598d3d7ef225 is 52, key is yyy/colfamily11:colfamily11/1731521753515/Put/seqid=0 2024-11-13T18:19:01,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741886_1062 (size=39263) 2024-11-13T18:19:01,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741886_1062 (size=39263) 2024-11-13T18:19:01,566 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/.tmp/colfamily11/add785c54a0049738830713c39642d34 2024-11-13T18:19:01,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741886_1062 (size=39263) 2024-11-13T18:19:01,578 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/.tmp/colfamily11/add785c54a0049738830713c39642d34 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/colfamily11/add785c54a0049738830713c39642d34 2024-11-13T18:19:01,582 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 02893942b5e22ded04197e4e8486e0dc, 1e715c3850b0001604b62543d4e17908, 37863ce38db650b86b62a2bccc5e7796, 3eefcb355d011c4ea65b0572629d5ef5, 575404eded2e7a15cbfa9186eea1d1b1, 5a59239e975bc7ad59b0ae9308416d51, 5aa68e8ac635bc9e7d6868f5f771281e, 5c39d6007aedf97e4c4a94511fe061ef, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677, fb720a25eb86c541f9c1f814843c651b, fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:01,584 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 09ff4ab7a9d0091782b93c194b270c43, 1588230740, 1b91248305f9bd170badfc1ac16c2f2b, 272983331631897f6cfeac3ce6bd7466, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:01,588 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/colfamily11/add785c54a0049738830713c39642d34, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:01,590 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1e715c3850b0001604b62543d4e17908 in 128ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741887_1063 (size=39263) 2024-11-13T18:19:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741887_1063 (size=39263) 2024-11-13T18:19:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741887_1063 (size=39263) 2024-11-13T18:19:01,607 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=60 2024-11-13T18:19:01,608 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:19:01,608 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:19:01,608 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731521941384Running coprocessor pre-close hooks at 1731521941384Disabling compacts and flushes for region at 1731521941384Disabling writes for close at 1731521941384Obtaining lock to block concurrent updates at 1731521941385 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731521941385Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=12192, getHeapSize=17984, getOffHeapSize=0, getCellsCount=64 at 1731521941394 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731521941397 (+3 ms)Flushing 1588230740/info: creating writer at 1731521941397Flushing 1588230740/info: appending metadata at 1731521941423 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731521941423Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a77fb45: reopening flushed file at 1731521941506 (+83 ms)Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 141ms, sequenceid=79, compaction requested=false at 1731521941525 (+19 ms)Writing region close event to WAL at 1731521941560 (+35 ms)Running coprocessor post-close hooks at 1731521941607 (+47 ms)Closed at 1731521941608 (+1 ms) 2024-11-13T18:19:01,608 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:19:01,627 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1e715c3850b0001604b62543d4e17908/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:01,628 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e715c3850b0001604b62543d4e17908: Waiting for close lock at 1731521941462Running coprocessor pre-close hooks at 1731521941462Disabling compacts and flushes for region at 1731521941462Disabling writes for close at 1731521941462Obtaining lock to block concurrent updates at 1731521941462Preparing flush snapshotting stores in 1e715c3850b0001604b62543d4e17908 at 1731521941462Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521941463 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. at 1731521941465 (+2 ms)Flushing 1e715c3850b0001604b62543d4e17908/colfamily11: creating writer at 1731521941465Flushing 1e715c3850b0001604b62543d4e17908/colfamily11: appending metadata at 1731521941510 (+45 ms)Flushing 1e715c3850b0001604b62543d4e17908/colfamily11: closing flushed file at 1731521941510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab3c8a9: reopening flushed file at 1731521941574 (+64 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1e715c3850b0001604b62543d4e17908 in 128ms, sequenceid=5, compaction requested=false at 1731521941590 (+16 ms)Writing region close event to WAL at 1731521941614 (+24 ms)Running coprocessor post-close hooks at 1731521941627 (+13 ms)Closed at 1731521941627 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731521748491.1e715c3850b0001604b62543d4e17908. 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5aa68e8ac635bc9e7d6868f5f771281e, disabling compactions & flushes 2024-11-13T18:19:01,628 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. after waiting 0 ms 2024-11-13T18:19:01,628 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:19:01,628 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5aa68e8ac635bc9e7d6868f5f771281e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:01,662 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/.tmp/colfamily11/b6c587accc7e4f7ba79f82ff15ba8f1e is 52, key is jjj/colfamily11:colfamily11/1731521753134/Put/seqid=0 2024-11-13T18:19:01,669 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fda96cf025dc:41233 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T18:19:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741888_1064 (size=39263) 2024-11-13T18:19:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741888_1064 (size=39263) 2024-11-13T18:19:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741888_1064 (size=39263) 2024-11-13T18:19:01,717 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/.tmp/colfamily11/b6c587accc7e4f7ba79f82ff15ba8f1e 2024-11-13T18:19:01,725 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/.tmp/colfamily11/b6c587accc7e4f7ba79f82ff15ba8f1e as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/colfamily11/b6c587accc7e4f7ba79f82ff15ba8f1e 2024-11-13T18:19:01,732 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/colfamily11/b6c587accc7e4f7ba79f82ff15ba8f1e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:01,733 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5aa68e8ac635bc9e7d6868f5f771281e in 105ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:01,738 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5aa68e8ac635bc9e7d6868f5f771281e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:01,739 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5aa68e8ac635bc9e7d6868f5f771281e: Waiting for close lock at 1731521941628Running coprocessor pre-close hooks at 1731521941628Disabling compacts and flushes for region at 1731521941628Disabling writes for close at 1731521941628Obtaining lock to block concurrent updates at 1731521941628Preparing flush snapshotting stores in 5aa68e8ac635bc9e7d6868f5f771281e at 1731521941628Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521941629 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. at 1731521941630 (+1 ms)Flushing 5aa68e8ac635bc9e7d6868f5f771281e/colfamily11: creating writer at 1731521941630Flushing 5aa68e8ac635bc9e7d6868f5f771281e/colfamily11: appending metadata at 1731521941662 (+32 ms)Flushing 5aa68e8ac635bc9e7d6868f5f771281e/colfamily11: closing flushed file at 1731521941662Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119a0265: reopening flushed file at 1731521941724 (+62 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5aa68e8ac635bc9e7d6868f5f771281e in 105ms, sequenceid=5, compaction requested=false at 1731521941733 (+9 ms)Writing region close event to WAL at 1731521941734 (+1 ms)Running coprocessor post-close hooks at 1731521941739 (+5 ms)Closed at 1731521941739 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731521748491.5aa68e8ac635bc9e7d6868f5f771281e. 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 02893942b5e22ded04197e4e8486e0dc, disabling compactions & flushes 2024-11-13T18:19:01,739 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. after waiting 0 ms 2024-11-13T18:19:01,739 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:19:01,765 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/02893942b5e22ded04197e4e8486e0dc/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:01,766 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:19:01,766 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 02893942b5e22ded04197e4e8486e0dc: Waiting for close lock at 1731521941739Running coprocessor pre-close hooks at 1731521941739Disabling compacts and flushes for region at 1731521941739Disabling writes for close at 1731521941739Writing region close event to WAL at 1731521941745 (+6 ms)Running coprocessor post-close hooks at 1731521941766 (+21 ms)Closed at 1731521941766 2024-11-13T18:19:01,767 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731521748491.02893942b5e22ded04197e4e8486e0dc. 2024-11-13T18:19:01,767 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3eefcb355d011c4ea65b0572629d5ef5, disabling compactions & flushes 2024-11-13T18:19:01,767 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:19:01,767 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:19:01,767 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. after waiting 0 ms 2024-11-13T18:19:01,767 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:19:01,783 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 37863ce38db650b86b62a2bccc5e7796, 3eefcb355d011c4ea65b0572629d5ef5, 575404eded2e7a15cbfa9186eea1d1b1, 5a59239e975bc7ad59b0ae9308416d51, 5c39d6007aedf97e4c4a94511fe061ef, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677, fb720a25eb86c541f9c1f814843c651b, fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:01,784 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 09ff4ab7a9d0091782b93c194b270c43, 1b91248305f9bd170badfc1ac16c2f2b, 272983331631897f6cfeac3ce6bd7466, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:01,787 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3eefcb355d011c4ea65b0572629d5ef5/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:01,788 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3eefcb355d011c4ea65b0572629d5ef5: Waiting for close lock at 1731521941767Running coprocessor pre-close hooks at 1731521941767Disabling compacts and flushes for region at 1731521941767Disabling writes for close at 1731521941767Writing region close event to WAL at 1731521941770 (+3 ms)Running coprocessor post-close hooks at 1731521941788 (+18 ms)Closed at 1731521941788 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731521748491.3eefcb355d011c4ea65b0572629d5ef5. 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5c39d6007aedf97e4c4a94511fe061ef, disabling compactions & flushes 2024-11-13T18:19:01,789 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. after waiting 0 ms 2024-11-13T18:19:01,789 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:19:01,789 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5c39d6007aedf97e4c4a94511fe061ef 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:01,818 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/.tmp/colfamily11/77aa2ed9b64948f5b123dafadea7851b is 52, key is hhh/colfamily11:colfamily11/1731521753075/Put/seqid=0 2024-11-13T18:19:01,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741889_1065 (size=39263) 2024-11-13T18:19:01,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741889_1065 (size=39263) 2024-11-13T18:19:01,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741889_1065 (size=39263) 2024-11-13T18:19:01,845 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/.tmp/colfamily11/77aa2ed9b64948f5b123dafadea7851b 2024-11-13T18:19:01,853 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/.tmp/colfamily11/77aa2ed9b64948f5b123dafadea7851b as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/colfamily11/77aa2ed9b64948f5b123dafadea7851b 2024-11-13T18:19:01,860 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/colfamily11/77aa2ed9b64948f5b123dafadea7851b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:01,862 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5c39d6007aedf97e4c4a94511fe061ef in 72ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:01,892 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5c39d6007aedf97e4c4a94511fe061ef/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:01,894 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5c39d6007aedf97e4c4a94511fe061ef: Waiting for close lock at 1731521941789Running coprocessor pre-close hooks at 1731521941789Disabling compacts and flushes for region at 1731521941789Disabling writes for close at 1731521941789Obtaining lock to block concurrent updates at 1731521941789Preparing flush snapshotting stores in 5c39d6007aedf97e4c4a94511fe061ef at 1731521941789Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521941790 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. at 1731521941791 (+1 ms)Flushing 5c39d6007aedf97e4c4a94511fe061ef/colfamily11: creating writer at 1731521941791Flushing 5c39d6007aedf97e4c4a94511fe061ef/colfamily11: appending metadata at 1731521941817 (+26 ms)Flushing 5c39d6007aedf97e4c4a94511fe061ef/colfamily11: closing flushed file at 1731521941817Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2da6983: reopening flushed file at 1731521941852 (+35 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5c39d6007aedf97e4c4a94511fe061ef in 72ms, sequenceid=5, compaction requested=false at 1731521941862 (+10 ms)Writing region close event to WAL at 1731521941863 (+1 ms)Running coprocessor post-close hooks at 1731521941894 (+31 ms)Closed at 1731521941894 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731521748491.5c39d6007aedf97e4c4a94511fe061ef. 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 575404eded2e7a15cbfa9186eea1d1b1, disabling compactions & flushes 2024-11-13T18:19:01,894 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. after waiting 0 ms 2024-11-13T18:19:01,894 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:19:01,894 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 575404eded2e7a15cbfa9186eea1d1b1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:01,928 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/.tmp/colfamily11/1b64ed0cc172431aa3476ad72f95b5f9 is 52, key is ppp/colfamily11:colfamily11/1731521753232/Put/seqid=0 2024-11-13T18:19:01,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741890_1066 (size=39263) 2024-11-13T18:19:01,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741890_1066 (size=39263) 2024-11-13T18:19:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741890_1066 (size=39263) 2024-11-13T18:19:01,959 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/.tmp/colfamily11/1b64ed0cc172431aa3476ad72f95b5f9 2024-11-13T18:19:01,970 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/.tmp/colfamily11/1b64ed0cc172431aa3476ad72f95b5f9 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/colfamily11/1b64ed0cc172431aa3476ad72f95b5f9 2024-11-13T18:19:01,983 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 37863ce38db650b86b62a2bccc5e7796, 575404eded2e7a15cbfa9186eea1d1b1, 5a59239e975bc7ad59b0ae9308416d51, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677, fb720a25eb86c541f9c1f814843c651b, fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:01,985 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 09ff4ab7a9d0091782b93c194b270c43, 1b91248305f9bd170badfc1ac16c2f2b, 272983331631897f6cfeac3ce6bd7466, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:01,990 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/colfamily11/1b64ed0cc172431aa3476ad72f95b5f9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:01,991 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 575404eded2e7a15cbfa9186eea1d1b1 in 97ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:01,998 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/575404eded2e7a15cbfa9186eea1d1b1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:01,999 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:19:01,999 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 575404eded2e7a15cbfa9186eea1d1b1: Waiting for close lock at 1731521941894Running coprocessor pre-close hooks at 1731521941894Disabling compacts and flushes for region at 1731521941894Disabling writes for close at 1731521941894Obtaining lock to block concurrent updates at 1731521941894Preparing flush snapshotting stores in 575404eded2e7a15cbfa9186eea1d1b1 at 1731521941894Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521941895 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. at 1731521941901 (+6 ms)Flushing 575404eded2e7a15cbfa9186eea1d1b1/colfamily11: creating writer at 1731521941901Flushing 575404eded2e7a15cbfa9186eea1d1b1/colfamily11: appending metadata at 1731521941928 (+27 ms)Flushing 575404eded2e7a15cbfa9186eea1d1b1/colfamily11: closing flushed file at 1731521941928Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@206163ec: reopening flushed file at 1731521941969 (+41 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 575404eded2e7a15cbfa9186eea1d1b1 in 97ms, sequenceid=5, compaction requested=false at 1731521941991 (+22 ms)Writing region close event to WAL at 1731521941992 (+1 ms)Running coprocessor post-close hooks at 1731521941999 (+7 ms)Closed at 1731521941999 2024-11-13T18:19:01,999 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731521748491.575404eded2e7a15cbfa9186eea1d1b1. 2024-11-13T18:19:02,000 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb720a25eb86c541f9c1f814843c651b, disabling compactions & flushes 2024-11-13T18:19:02,000 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:19:02,000 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:19:02,000 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. after waiting 0 ms 2024-11-13T18:19:02,000 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:19:02,007 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/.tmp/colfamily11/c44c426d3b7241149f8f598d3d7ef225 2024-11-13T18:19:02,008 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb720a25eb86c541f9c1f814843c651b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:02,008 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb720a25eb86c541f9c1f814843c651b: Waiting for close lock at 1731521942000Running coprocessor pre-close hooks at 1731521942000Disabling compacts and flushes for region at 1731521942000Disabling writes for close at 1731521942000Writing region close event to WAL at 1731521942001 (+1 ms)Running coprocessor post-close hooks at 1731521942008 (+7 ms)Closed at 1731521942008 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731521748491.fb720a25eb86c541f9c1f814843c651b. 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd437c88e4af73574ce0ce0882aca142, disabling compactions & flushes 2024-11-13T18:19:02,009 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. after waiting 0 ms 2024-11-13T18:19:02,009 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:19:02,009 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fd437c88e4af73574ce0ce0882aca142 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,019 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/.tmp/colfamily11/c44c426d3b7241149f8f598d3d7ef225 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/colfamily11/c44c426d3b7241149f8f598d3d7ef225 2024-11-13T18:19:02,033 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/colfamily11/c44c426d3b7241149f8f598d3d7ef225, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,034 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 272983331631897f6cfeac3ce6bd7466 in 571ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,051 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/.tmp/colfamily11/9318b9461af640ec97eae7f1d00e1249 is 52, key is aaa/colfamily11:colfamily11/1731521752880/Put/seqid=0 2024-11-13T18:19:02,086 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/272983331631897f6cfeac3ce6bd7466/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,088 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:19:02,088 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 272983331631897f6cfeac3ce6bd7466: Waiting for close lock at 1731521941463Running coprocessor pre-close hooks at 1731521941463Disabling compacts and flushes for region at 1731521941463Disabling writes for close at 1731521941463Obtaining lock to block concurrent updates at 1731521941464 (+1 ms)Preparing flush snapshotting stores in 272983331631897f6cfeac3ce6bd7466 at 1731521941464Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521941464Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. at 1731521941465 (+1 ms)Flushing 272983331631897f6cfeac3ce6bd7466/colfamily11: creating writer at 1731521941465Flushing 272983331631897f6cfeac3ce6bd7466/colfamily11: appending metadata at 1731521941525 (+60 ms)Flushing 272983331631897f6cfeac3ce6bd7466/colfamily11: closing flushed file at 1731521941525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b345ca3: reopening flushed file at 1731521942017 (+492 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 272983331631897f6cfeac3ce6bd7466 in 571ms, sequenceid=5, compaction requested=false at 1731521942034 (+17 ms)Writing region close event to WAL at 1731521942055 (+21 ms)Running coprocessor post-close hooks at 1731521942088 (+33 ms)Closed at 1731521942088 2024-11-13T18:19:02,088 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731521748491.272983331631897f6cfeac3ce6bd7466. 2024-11-13T18:19:02,088 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 09ff4ab7a9d0091782b93c194b270c43, disabling compactions & flushes 2024-11-13T18:19:02,088 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:19:02,088 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:19:02,088 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. after waiting 0 ms 2024-11-13T18:19:02,089 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:19:02,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741891_1067 (size=39263) 2024-11-13T18:19:02,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741891_1067 (size=39263) 2024-11-13T18:19:02,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741891_1067 (size=39263) 2024-11-13T18:19:02,152 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/.tmp/colfamily11/9318b9461af640ec97eae7f1d00e1249 2024-11-13T18:19:02,175 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/09ff4ab7a9d0091782b93c194b270c43/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:02,176 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:19:02,176 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 09ff4ab7a9d0091782b93c194b270c43: Waiting for close lock at 1731521942088Running coprocessor pre-close hooks at 1731521942088Disabling compacts and flushes for region at 1731521942088Disabling writes for close at 1731521942089 (+1 ms)Writing region close event to WAL at 1731521942151 (+62 ms)Running coprocessor post-close hooks at 1731521942176 (+25 ms)Closed at 1731521942176 2024-11-13T18:19:02,176 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731521748491.09ff4ab7a9d0091782b93c194b270c43. 2024-11-13T18:19:02,177 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b196a2f44f4f468f8bb9a2e59b301747, disabling compactions & flushes 2024-11-13T18:19:02,177 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:19:02,177 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:19:02,177 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. after waiting 0 ms 2024-11-13T18:19:02,177 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:19:02,177 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b196a2f44f4f468f8bb9a2e59b301747 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,182 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/.tmp/colfamily11/9318b9461af640ec97eae7f1d00e1249 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/colfamily11/9318b9461af640ec97eae7f1d00e1249 2024-11-13T18:19:02,183 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 37863ce38db650b86b62a2bccc5e7796, 5a59239e975bc7ad59b0ae9308416d51, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677, fd437c88e4af73574ce0ce0882aca142 2024-11-13T18:19:02,188 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 1b91248305f9bd170badfc1ac16c2f2b, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:02,193 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/colfamily11/9318b9461af640ec97eae7f1d00e1249, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,201 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fd437c88e4af73574ce0ce0882aca142 in 191ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,213 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:19:02,213 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:19:02,236 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/.tmp/colfamily11/2f5e57e202d549c39a4424bd7cec7f24 is 52, key is www/colfamily11:colfamily11/1731521753467/Put/seqid=0 2024-11-13T18:19:02,243 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd437c88e4af73574ce0ce0882aca142/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,244 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:19:02,244 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd437c88e4af73574ce0ce0882aca142: Waiting for close lock at 1731521942009Running coprocessor pre-close hooks at 1731521942009Disabling compacts and flushes for region at 1731521942009Disabling writes for close at 1731521942009Obtaining lock to block concurrent updates at 1731521942009Preparing flush snapshotting stores in fd437c88e4af73574ce0ce0882aca142 at 1731521942009Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942010 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. at 1731521942011 (+1 ms)Flushing fd437c88e4af73574ce0ce0882aca142/colfamily11: creating writer at 1731521942011Flushing fd437c88e4af73574ce0ce0882aca142/colfamily11: appending metadata at 1731521942050 (+39 ms)Flushing fd437c88e4af73574ce0ce0882aca142/colfamily11: closing flushed file at 1731521942050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e04de: reopening flushed file at 1731521942177 (+127 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fd437c88e4af73574ce0ce0882aca142 in 191ms, sequenceid=5, compaction requested=false at 1731521942201 (+24 ms)Writing region close event to WAL at 1731521942226 (+25 ms)Running coprocessor post-close hooks at 1731521942244 (+18 ms)Closed at 1731521942244 2024-11-13T18:19:02,244 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731521748491.fd437c88e4af73574ce0ce0882aca142. 2024-11-13T18:19:02,244 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37863ce38db650b86b62a2bccc5e7796, disabling compactions & flushes 2024-11-13T18:19:02,244 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:19:02,244 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:19:02,245 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. after waiting 0 ms 2024-11-13T18:19:02,245 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:19:02,245 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 37863ce38db650b86b62a2bccc5e7796 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,260 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:19:02,261 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:19:02,285 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/.tmp/colfamily11/ce152fa7c2b146858840d1860069d151 is 52, key is nnn/colfamily11:colfamily11/1731521753187/Put/seqid=0 2024-11-13T18:19:02,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741892_1068 (size=39263) 2024-11-13T18:19:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741892_1068 (size=39263) 2024-11-13T18:19:02,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741892_1068 (size=39263) 2024-11-13T18:19:02,381 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/.tmp/colfamily11/2f5e57e202d549c39a4424bd7cec7f24 2024-11-13T18:19:02,383 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-13T18:19:02,384 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1325): Online Regions={37863ce38db650b86b62a2bccc5e7796=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., 644025ac5ad44a7ca4a26c89d6611d8b=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., 5a59239e975bc7ad59b0ae9308416d51=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., cf5aa9737456e8265a9b1e93d818d677=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677.} 2024-11-13T18:19:02,384 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 37863ce38db650b86b62a2bccc5e7796, 5a59239e975bc7ad59b0ae9308416d51, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:19:02,389 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-13T18:19:02,389 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 00a4d576f4c41497f18224865b73403a, 1b91248305f9bd170badfc1ac16c2f2b, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 64b5086eb7020ccba10769de2eaf3d9b, 801d0f207e45efea50135a16f5fcbc99, b196a2f44f4f468f8bb9a2e59b301747, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:02,393 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/.tmp/colfamily11/2f5e57e202d549c39a4424bd7cec7f24 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/colfamily11/2f5e57e202d549c39a4424bd7cec7f24 2024-11-13T18:19:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741893_1069 (size=39263) 2024-11-13T18:19:02,408 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/colfamily11/2f5e57e202d549c39a4424bd7cec7f24, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741893_1069 (size=39263) 2024-11-13T18:19:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741893_1069 (size=39263) 2024-11-13T18:19:02,410 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b196a2f44f4f468f8bb9a2e59b301747 in 233ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,428 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/.tmp/colfamily11/ce152fa7c2b146858840d1860069d151 2024-11-13T18:19:02,449 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/.tmp/colfamily11/ce152fa7c2b146858840d1860069d151 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/colfamily11/ce152fa7c2b146858840d1860069d151 2024-11-13T18:19:02,454 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b196a2f44f4f468f8bb9a2e59b301747/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,454 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:19:02,454 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b196a2f44f4f468f8bb9a2e59b301747: Waiting for close lock at 1731521942177Running coprocessor pre-close hooks at 1731521942177Disabling compacts and flushes for region at 1731521942177Disabling writes for close at 1731521942177Obtaining lock to block concurrent updates at 1731521942177Preparing flush snapshotting stores in b196a2f44f4f468f8bb9a2e59b301747 at 1731521942177Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942177Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. at 1731521942178 (+1 ms)Flushing b196a2f44f4f468f8bb9a2e59b301747/colfamily11: creating writer at 1731521942178Flushing b196a2f44f4f468f8bb9a2e59b301747/colfamily11: appending metadata at 1731521942235 (+57 ms)Flushing b196a2f44f4f468f8bb9a2e59b301747/colfamily11: closing flushed file at 1731521942235Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@460d9a8d: reopening flushed file at 1731521942392 (+157 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b196a2f44f4f468f8bb9a2e59b301747 in 233ms, sequenceid=5, compaction requested=false at 1731521942410 (+18 ms)Writing region close event to WAL at 1731521942437 (+27 ms)Running coprocessor post-close hooks at 1731521942454 (+17 ms)Closed at 1731521942454 2024-11-13T18:19:02,455 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731521748491.b196a2f44f4f468f8bb9a2e59b301747. 2024-11-13T18:19:02,455 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00a4d576f4c41497f18224865b73403a, disabling compactions & flushes 2024-11-13T18:19:02,455 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:19:02,455 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:19:02,455 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. after waiting 0 ms 2024-11-13T18:19:02,455 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:19:02,455 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 00a4d576f4c41497f18224865b73403a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,469 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/colfamily11/ce152fa7c2b146858840d1860069d151, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,471 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 37863ce38db650b86b62a2bccc5e7796 in 226ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,483 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/.tmp/colfamily11/867ee5cc26ef48e78fae63d9c6aad5f5 is 52, key is ddd/colfamily11:colfamily11/1731521752957/Put/seqid=0 2024-11-13T18:19:02,489 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/37863ce38db650b86b62a2bccc5e7796/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,489 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37863ce38db650b86b62a2bccc5e7796: Waiting for close lock at 1731521942244Running coprocessor pre-close hooks at 1731521942244Disabling compacts and flushes for region at 1731521942244Disabling writes for close at 1731521942245 (+1 ms)Obtaining lock to block concurrent updates at 1731521942245Preparing flush snapshotting stores in 37863ce38db650b86b62a2bccc5e7796 at 1731521942245Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942245Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. at 1731521942249 (+4 ms)Flushing 37863ce38db650b86b62a2bccc5e7796/colfamily11: creating writer at 1731521942249Flushing 37863ce38db650b86b62a2bccc5e7796/colfamily11: appending metadata at 1731521942284 (+35 ms)Flushing 37863ce38db650b86b62a2bccc5e7796/colfamily11: closing flushed file at 1731521942284Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e93fff2: reopening flushed file at 1731521942445 (+161 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 37863ce38db650b86b62a2bccc5e7796 in 226ms, sequenceid=5, compaction requested=false at 1731521942471 (+26 ms)Writing region close event to WAL at 1731521942475 (+4 ms)Running coprocessor post-close hooks at 1731521942489 (+14 ms)Closed at 1731521942489 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731521748491.37863ce38db650b86b62a2bccc5e7796. 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 644025ac5ad44a7ca4a26c89d6611d8b, disabling compactions & flushes 2024-11-13T18:19:02,490 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. after waiting 0 ms 2024-11-13T18:19:02,490 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:19:02,490 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 644025ac5ad44a7ca4a26c89d6611d8b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,515 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/.tmp/colfamily11/e7a572dc4ec24cd0b5d9e9da1c666dac is 52, key is ttt/colfamily11:colfamily11/1731521753320/Put/seqid=0 2024-11-13T18:19:02,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741894_1070 (size=39263) 2024-11-13T18:19:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741894_1070 (size=39263) 2024-11-13T18:19:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741894_1070 (size=39263) 2024-11-13T18:19:02,526 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/.tmp/colfamily11/867ee5cc26ef48e78fae63d9c6aad5f5 2024-11-13T18:19:02,533 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/.tmp/colfamily11/867ee5cc26ef48e78fae63d9c6aad5f5 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/colfamily11/867ee5cc26ef48e78fae63d9c6aad5f5 2024-11-13T18:19:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741895_1071 (size=39263) 2024-11-13T18:19:02,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741895_1071 (size=39263) 2024-11-13T18:19:02,540 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/.tmp/colfamily11/e7a572dc4ec24cd0b5d9e9da1c666dac 2024-11-13T18:19:02,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741895_1071 (size=39263) 2024-11-13T18:19:02,544 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/colfamily11/867ee5cc26ef48e78fae63d9c6aad5f5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,545 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 00a4d576f4c41497f18224865b73403a in 90ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,551 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00a4d576f4c41497f18224865b73403a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,551 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00a4d576f4c41497f18224865b73403a: Waiting for close lock at 1731521942455Running coprocessor pre-close hooks at 1731521942455Disabling compacts and flushes for region at 1731521942455Disabling writes for close at 1731521942455Obtaining lock to block concurrent updates at 1731521942455Preparing flush snapshotting stores in 00a4d576f4c41497f18224865b73403a at 1731521942455Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942455Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. at 1731521942457 (+2 ms)Flushing 00a4d576f4c41497f18224865b73403a/colfamily11: creating writer at 1731521942457Flushing 00a4d576f4c41497f18224865b73403a/colfamily11: appending metadata at 1731521942481 (+24 ms)Flushing 00a4d576f4c41497f18224865b73403a/colfamily11: closing flushed file at 1731521942481Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f809b81: reopening flushed file at 1731521942532 (+51 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 00a4d576f4c41497f18224865b73403a in 90ms, sequenceid=5, compaction requested=false at 1731521942545 (+13 ms)Writing region close event to WAL at 1731521942546 (+1 ms)Running coprocessor post-close hooks at 1731521942551 (+5 ms)Closed at 1731521942551 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731521748491.00a4d576f4c41497f18224865b73403a. 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64b5086eb7020ccba10769de2eaf3d9b, disabling compactions & flushes 2024-11-13T18:19:02,552 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. after waiting 0 ms 2024-11-13T18:19:02,552 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:19:02,556 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64b5086eb7020ccba10769de2eaf3d9b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:02,557 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:19:02,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64b5086eb7020ccba10769de2eaf3d9b: Waiting for close lock at 1731521942552Running coprocessor pre-close hooks at 1731521942552Disabling compacts and flushes for region at 1731521942552Disabling writes for close at 1731521942552Writing region close event to WAL at 1731521942553 (+1 ms)Running coprocessor post-close hooks at 1731521942557 (+4 ms)Closed at 1731521942557 2024-11-13T18:19:02,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731521748491.64b5086eb7020ccba10769de2eaf3d9b. 2024-11-13T18:19:02,558 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 27ae2919589ed9f92a8ad04abb8e24d5, disabling compactions & flushes 2024-11-13T18:19:02,558 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:19:02,558 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:19:02,558 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. after waiting 0 ms 2024-11-13T18:19:02,558 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:19:02,558 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 27ae2919589ed9f92a8ad04abb8e24d5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,565 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/.tmp/colfamily11/e7a572dc4ec24cd0b5d9e9da1c666dac as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/colfamily11/e7a572dc4ec24cd0b5d9e9da1c666dac 2024-11-13T18:19:02,576 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/colfamily11/e7a572dc4ec24cd0b5d9e9da1c666dac, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,578 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 644025ac5ad44a7ca4a26c89d6611d8b in 87ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,584 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on 5a59239e975bc7ad59b0ae9308416d51, 644025ac5ad44a7ca4a26c89d6611d8b, cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:19:02,587 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/644025ac5ad44a7ca4a26c89d6611d8b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,588 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 644025ac5ad44a7ca4a26c89d6611d8b: Waiting for close lock at 1731521942490Running coprocessor pre-close hooks at 1731521942490Disabling compacts and flushes for region at 1731521942490Disabling writes for close at 1731521942490Obtaining lock to block concurrent updates at 1731521942490Preparing flush snapshotting stores in 644025ac5ad44a7ca4a26c89d6611d8b at 1731521942490Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942491 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. at 1731521942491Flushing 644025ac5ad44a7ca4a26c89d6611d8b/colfamily11: creating writer at 1731521942492 (+1 ms)Flushing 644025ac5ad44a7ca4a26c89d6611d8b/colfamily11: appending metadata at 1731521942514 (+22 ms)Flushing 644025ac5ad44a7ca4a26c89d6611d8b/colfamily11: closing flushed file at 1731521942515 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f72eaf6: reopening flushed file at 1731521942558 (+43 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 644025ac5ad44a7ca4a26c89d6611d8b in 87ms, sequenceid=5, compaction requested=false at 1731521942578 (+20 ms)Writing region close event to WAL at 1731521942582 (+4 ms)Running coprocessor post-close hooks at 1731521942588 (+6 ms)Closed at 1731521942588 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731521748491.644025ac5ad44a7ca4a26c89d6611d8b. 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a59239e975bc7ad59b0ae9308416d51, disabling compactions & flushes 2024-11-13T18:19:02,588 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. after waiting 0 ms 2024-11-13T18:19:02,588 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:19:02,588 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5a59239e975bc7ad59b0ae9308416d51 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,589 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 27ae2919589ed9f92a8ad04abb8e24d5, 38c892b7353daa24a9b6c917f1aa0a89, 400b285d3ec5b0d43bcd37cdfa6b8f90, 801d0f207e45efea50135a16f5fcbc99, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:02,595 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/.tmp/colfamily11/439d97385198484a93a021de96ff4cb4 is 52, key is mmm/colfamily11:colfamily11/1731521753249/Put/seqid=0 2024-11-13T18:19:02,611 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/.tmp/colfamily11/2ad669d65a594c088505dc140352eb82 is 52, key is vvv/colfamily11:colfamily11/1731521753405/Put/seqid=0 2024-11-13T18:19:02,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741896_1072 (size=39263) 2024-11-13T18:19:02,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741896_1072 (size=39263) 2024-11-13T18:19:02,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741896_1072 (size=39263) 2024-11-13T18:19:02,613 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/.tmp/colfamily11/439d97385198484a93a021de96ff4cb4 2024-11-13T18:19:02,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/.tmp/colfamily11/439d97385198484a93a021de96ff4cb4 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/colfamily11/439d97385198484a93a021de96ff4cb4 2024-11-13T18:19:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741897_1073 (size=39263) 2024-11-13T18:19:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741897_1073 (size=39263) 2024-11-13T18:19:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741897_1073 (size=39263) 2024-11-13T18:19:02,646 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/.tmp/colfamily11/2ad669d65a594c088505dc140352eb82 2024-11-13T18:19:02,653 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/.tmp/colfamily11/2ad669d65a594c088505dc140352eb82 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/colfamily11/2ad669d65a594c088505dc140352eb82 2024-11-13T18:19:02,655 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/colfamily11/439d97385198484a93a021de96ff4cb4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,656 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 27ae2919589ed9f92a8ad04abb8e24d5 in 98ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,659 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/colfamily11/2ad669d65a594c088505dc140352eb82, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,660 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5a59239e975bc7ad59b0ae9308416d51 in 72ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,673 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/27ae2919589ed9f92a8ad04abb8e24d5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,674 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:19:02,674 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 27ae2919589ed9f92a8ad04abb8e24d5: Waiting for close lock at 1731521942558Running coprocessor pre-close hooks at 1731521942558Disabling compacts and flushes for region at 1731521942558Disabling writes for close at 1731521942558Obtaining lock to block concurrent updates at 1731521942558Preparing flush snapshotting stores in 27ae2919589ed9f92a8ad04abb8e24d5 at 1731521942558Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942558Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. at 1731521942561 (+3 ms)Flushing 27ae2919589ed9f92a8ad04abb8e24d5/colfamily11: creating writer at 1731521942561Flushing 27ae2919589ed9f92a8ad04abb8e24d5/colfamily11: appending metadata at 1731521942594 (+33 ms)Flushing 27ae2919589ed9f92a8ad04abb8e24d5/colfamily11: closing flushed file at 1731521942594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c9390e1: reopening flushed file at 1731521942641 (+47 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 27ae2919589ed9f92a8ad04abb8e24d5 in 98ms, sequenceid=5, compaction requested=false at 1731521942656 (+15 ms)Writing region close event to WAL at 1731521942657 (+1 ms)Running coprocessor post-close hooks at 1731521942674 (+17 ms)Closed at 1731521942674 2024-11-13T18:19:02,674 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731521748491.27ae2919589ed9f92a8ad04abb8e24d5. 2024-11-13T18:19:02,674 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 400b285d3ec5b0d43bcd37cdfa6b8f90, disabling compactions & flushes 2024-11-13T18:19:02,674 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:19:02,675 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:19:02,675 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. after waiting 0 ms 2024-11-13T18:19:02,675 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:19:02,677 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5a59239e975bc7ad59b0ae9308416d51/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,678 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a59239e975bc7ad59b0ae9308416d51: Waiting for close lock at 1731521942588Running coprocessor pre-close hooks at 1731521942588Disabling compacts and flushes for region at 1731521942588Disabling writes for close at 1731521942588Obtaining lock to block concurrent updates at 1731521942588Preparing flush snapshotting stores in 5a59239e975bc7ad59b0ae9308416d51 at 1731521942588Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942589 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. at 1731521942589Flushing 5a59239e975bc7ad59b0ae9308416d51/colfamily11: creating writer at 1731521942590 (+1 ms)Flushing 5a59239e975bc7ad59b0ae9308416d51/colfamily11: appending metadata at 1731521942610 (+20 ms)Flushing 5a59239e975bc7ad59b0ae9308416d51/colfamily11: closing flushed file at 1731521942610Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e2065d4: reopening flushed file at 1731521942652 (+42 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5a59239e975bc7ad59b0ae9308416d51 in 72ms, sequenceid=5, compaction requested=false at 1731521942660 (+8 ms)Writing region close event to WAL at 1731521942669 (+9 ms)Running coprocessor post-close hooks at 1731521942678 (+9 ms)Closed at 1731521942678 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731521748491.5a59239e975bc7ad59b0ae9308416d51. 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf5aa9737456e8265a9b1e93d818d677, disabling compactions & flushes 2024-11-13T18:19:02,678 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. after waiting 0 ms 2024-11-13T18:19:02,678 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:19:02,678 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cf5aa9737456e8265a9b1e93d818d677 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-13T18:19:02,706 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/.tmp/colfamily11/2349c209cdd44ff58bcf982030461f2c is 52, key is zzz/colfamily11:colfamily11/1731521753413/Put/seqid=0 2024-11-13T18:19:02,713 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/400b285d3ec5b0d43bcd37cdfa6b8f90/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T18:19:02,717 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:19:02,717 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 400b285d3ec5b0d43bcd37cdfa6b8f90: Waiting for close lock at 1731521942674Running coprocessor pre-close hooks at 1731521942674Disabling compacts and flushes for region at 1731521942674Disabling writes for close at 1731521942675 (+1 ms)Writing region close event to WAL at 1731521942695 (+20 ms)Running coprocessor post-close hooks at 1731521942717 (+22 ms)Closed at 1731521942717 2024-11-13T18:19:02,717 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731521748491.400b285d3ec5b0d43bcd37cdfa6b8f90. 2024-11-13T18:19:02,718 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5bab85a9454bcf58ade3574335c3bd1, disabling compactions & flushes 2024-11-13T18:19:02,718 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:19:02,718 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:19:02,718 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. after waiting 0 ms 2024-11-13T18:19:02,718 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:19:02,718 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e5bab85a9454bcf58ade3574335c3bd1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,757 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/.tmp/colfamily11/308a051469964a778373b0abac687c7a is 52, key is kkk/colfamily11:colfamily11/1731521753167/Put/seqid=0 2024-11-13T18:19:02,785 DEBUG [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1351): Waiting on cf5aa9737456e8265a9b1e93d818d677 2024-11-13T18:19:02,789 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89, 801d0f207e45efea50135a16f5fcbc99, dae4cf552a1afaf0b041668499d99e20, e5bab85a9454bcf58ade3574335c3bd1 2024-11-13T18:19:02,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741898_1074 (size=4857) 2024-11-13T18:19:02,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741898_1074 (size=4857) 2024-11-13T18:19:02,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741898_1074 (size=4857) 2024-11-13T18:19:02,797 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/.tmp/colfamily11/2349c209cdd44ff58bcf982030461f2c 2024-11-13T18:19:02,814 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/.tmp/colfamily11/2349c209cdd44ff58bcf982030461f2c as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/colfamily11/2349c209cdd44ff58bcf982030461f2c 2024-11-13T18:19:02,832 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/colfamily11/2349c209cdd44ff58bcf982030461f2c, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T18:19:02,850 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for cf5aa9737456e8265a9b1e93d818d677 in 171ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741899_1075 (size=39263) 2024-11-13T18:19:02,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741899_1075 (size=39263) 2024-11-13T18:19:02,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741899_1075 (size=39263) 2024-11-13T18:19:02,870 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/.tmp/colfamily11/308a051469964a778373b0abac687c7a 2024-11-13T18:19:02,895 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/.tmp/colfamily11/308a051469964a778373b0abac687c7a as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/colfamily11/308a051469964a778373b0abac687c7a 2024-11-13T18:19:02,906 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf5aa9737456e8265a9b1e93d818d677/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,908 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:19:02,908 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf5aa9737456e8265a9b1e93d818d677: Waiting for close lock at 1731521942678Running coprocessor pre-close hooks at 1731521942678Disabling compacts and flushes for region at 1731521942678Disabling writes for close at 1731521942678Obtaining lock to block concurrent updates at 1731521942678Preparing flush snapshotting stores in cf5aa9737456e8265a9b1e93d818d677 at 1731521942678Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731521942679 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. at 1731521942679Flushing cf5aa9737456e8265a9b1e93d818d677/colfamily11: creating writer at 1731521942680 (+1 ms)Flushing cf5aa9737456e8265a9b1e93d818d677/colfamily11: appending metadata at 1731521942706 (+26 ms)Flushing cf5aa9737456e8265a9b1e93d818d677/colfamily11: closing flushed file at 1731521942706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d6b51f2: reopening flushed file at 1731521942808 (+102 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for cf5aa9737456e8265a9b1e93d818d677 in 171ms, sequenceid=5, compaction requested=false at 1731521942850 (+42 ms)Writing region close event to WAL at 1731521942884 (+34 ms)Running coprocessor post-close hooks at 1731521942908 (+24 ms)Closed at 1731521942908 2024-11-13T18:19:02,908 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731521748491.cf5aa9737456e8265a9b1e93d818d677. 2024-11-13T18:19:02,910 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/colfamily11/308a051469964a778373b0abac687c7a, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:02,911 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e5bab85a9454bcf58ade3574335c3bd1 in 193ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:02,965 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e5bab85a9454bcf58ade3574335c3bd1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:02,966 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:19:02,966 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5bab85a9454bcf58ade3574335c3bd1: Waiting for close lock at 1731521942718Running coprocessor pre-close hooks at 1731521942718Disabling compacts and flushes for region at 1731521942718Disabling writes for close at 1731521942718Obtaining lock to block concurrent updates at 1731521942718Preparing flush snapshotting stores in e5bab85a9454bcf58ade3574335c3bd1 at 1731521942718Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942718Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. at 1731521942719 (+1 ms)Flushing e5bab85a9454bcf58ade3574335c3bd1/colfamily11: creating writer at 1731521942719Flushing e5bab85a9454bcf58ade3574335c3bd1/colfamily11: appending metadata at 1731521942756 (+37 ms)Flushing e5bab85a9454bcf58ade3574335c3bd1/colfamily11: closing flushed file at 1731521942756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44ad5367: reopening flushed file at 1731521942888 (+132 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e5bab85a9454bcf58ade3574335c3bd1 in 193ms, sequenceid=5, compaction requested=false at 1731521942911 (+23 ms)Writing region close event to WAL at 1731521942948 (+37 ms)Running coprocessor post-close hooks at 1731521942966 (+18 ms)Closed at 1731521942966 2024-11-13T18:19:02,966 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731521748491.e5bab85a9454bcf58ade3574335c3bd1. 2024-11-13T18:19:02,966 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dae4cf552a1afaf0b041668499d99e20, disabling compactions & flushes 2024-11-13T18:19:02,966 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:19:02,966 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:19:02,967 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. after waiting 0 ms 2024-11-13T18:19:02,967 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:19:02,967 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dae4cf552a1afaf0b041668499d99e20 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:02,985 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,46791,1731521745104; all regions closed. 2024-11-13T18:19:02,992 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89, 801d0f207e45efea50135a16f5fcbc99, dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:19:03,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741834_1010 (size=316519) 2024-11-13T18:19:03,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741834_1010 (size=316519) 2024-11-13T18:19:03,022 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/.tmp/colfamily11/a80617627931457b95110b50bc2cafb4 is 52, key is qqq/colfamily11:colfamily11/1731521753329/Put/seqid=0 2024-11-13T18:19:03,026 DEBUG [RS:1;fda96cf025dc:46791 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C46791%2C1731521745104:(num 1731521747390) 2024-11-13T18:19:03,027 DEBUG [RS:1;fda96cf025dc:46791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:19:03,027 INFO [RS:1;fda96cf025dc:46791 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:19:03,028 INFO [RS:1;fda96cf025dc:46791 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46791 2024-11-13T18:19:03,028 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:19:03,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:19:03,036 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,46791,1731521745104] 2024-11-13T18:19:03,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,46791,1731521745104 2024-11-13T18:19:03,036 INFO [RS:1;fda96cf025dc:46791 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:19:03,038 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,46791,1731521745104 already deleted, retry=false 2024-11-13T18:19:03,038 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,46791,1731521745104 expired; onlineServers=1 2024-11-13T18:19:03,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741900_1076 (size=39263) 2024-11-13T18:19:03,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741900_1076 (size=39263) 2024-11-13T18:19:03,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741900_1076 (size=39263) 2024-11-13T18:19:03,096 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/.tmp/colfamily11/a80617627931457b95110b50bc2cafb4 2024-11-13T18:19:03,140 INFO [RS:1;fda96cf025dc:46791 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:19:03,140 INFO [RS:1;fda96cf025dc:46791 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,46791,1731521745104; zookeeper connection closed. 2024-11-13T18:19:03,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:03,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x1003a5b1ab20002, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:03,148 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4350164 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4350164 2024-11-13T18:19:03,152 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/.tmp/colfamily11/a80617627931457b95110b50bc2cafb4 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/colfamily11/a80617627931457b95110b50bc2cafb4 2024-11-13T18:19:03,179 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/colfamily11/a80617627931457b95110b50bc2cafb4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:03,186 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dae4cf552a1afaf0b041668499d99e20 in 219ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:03,192 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89, 801d0f207e45efea50135a16f5fcbc99, dae4cf552a1afaf0b041668499d99e20 2024-11-13T18:19:03,229 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dae4cf552a1afaf0b041668499d99e20/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:03,230 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dae4cf552a1afaf0b041668499d99e20: Waiting for close lock at 1731521942966Running coprocessor pre-close hooks at 1731521942966Disabling compacts and flushes for region at 1731521942966Disabling writes for close at 1731521942967 (+1 ms)Obtaining lock to block concurrent updates at 1731521942967Preparing flush snapshotting stores in dae4cf552a1afaf0b041668499d99e20 at 1731521942967Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521942967Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. at 1731521942968 (+1 ms)Flushing dae4cf552a1afaf0b041668499d99e20/colfamily11: creating writer at 1731521942968Flushing dae4cf552a1afaf0b041668499d99e20/colfamily11: appending metadata at 1731521943021 (+53 ms)Flushing dae4cf552a1afaf0b041668499d99e20/colfamily11: closing flushed file at 1731521943021Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e023424: reopening flushed file at 1731521943149 (+128 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dae4cf552a1afaf0b041668499d99e20 in 219ms, sequenceid=5, compaction requested=false at 1731521943187 (+38 ms)Writing region close event to WAL at 1731521943208 (+21 ms)Running coprocessor post-close hooks at 1731521943230 (+22 ms)Closed at 1731521943230 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731521748491.dae4cf552a1afaf0b041668499d99e20. 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 801d0f207e45efea50135a16f5fcbc99, disabling compactions & flushes 2024-11-13T18:19:03,231 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. after waiting 0 ms 2024-11-13T18:19:03,231 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:19:03,231 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 801d0f207e45efea50135a16f5fcbc99 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:03,260 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/.tmp/colfamily11/5571c1914f5a47ec803db9d659deda59 is 52, key is ggg/colfamily11:colfamily11/1731521753097/Put/seqid=0 2024-11-13T18:19:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741901_1077 (size=39263) 2024-11-13T18:19:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741901_1077 (size=39263) 2024-11-13T18:19:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741901_1077 (size=39263) 2024-11-13T18:19:03,356 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/.tmp/colfamily11/5571c1914f5a47ec803db9d659deda59 2024-11-13T18:19:03,396 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-13T18:19:03,396 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1325): Online Regions={801d0f207e45efea50135a16f5fcbc99=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., 1b91248305f9bd170badfc1ac16c2f2b=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., 38c892b7353daa24a9b6c917f1aa0a89=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89.} 2024-11-13T18:19:03,396 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89, 801d0f207e45efea50135a16f5fcbc99 2024-11-13T18:19:03,408 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/.tmp/colfamily11/5571c1914f5a47ec803db9d659deda59 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/colfamily11/5571c1914f5a47ec803db9d659deda59 2024-11-13T18:19:03,459 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/colfamily11/5571c1914f5a47ec803db9d659deda59, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:03,464 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 801d0f207e45efea50135a16f5fcbc99 in 233ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:03,532 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/801d0f207e45efea50135a16f5fcbc99/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:03,557 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 801d0f207e45efea50135a16f5fcbc99: Waiting for close lock at 1731521943231Running coprocessor pre-close hooks at 1731521943231Disabling compacts and flushes for region at 1731521943231Disabling writes for close at 1731521943231Obtaining lock to block concurrent updates at 1731521943231Preparing flush snapshotting stores in 801d0f207e45efea50135a16f5fcbc99 at 1731521943231Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521943232 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. at 1731521943239 (+7 ms)Flushing 801d0f207e45efea50135a16f5fcbc99/colfamily11: creating writer at 1731521943239Flushing 801d0f207e45efea50135a16f5fcbc99/colfamily11: appending metadata at 1731521943260 (+21 ms)Flushing 801d0f207e45efea50135a16f5fcbc99/colfamily11: closing flushed file at 1731521943260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a4d2aea: reopening flushed file at 1731521943397 (+137 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 801d0f207e45efea50135a16f5fcbc99 in 233ms, sequenceid=5, compaction requested=false at 1731521943464 (+67 ms)Writing region close event to WAL at 1731521943481 (+17 ms)Running coprocessor post-close hooks at 1731521943556 (+75 ms)Closed at 1731521943556 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731521748491.801d0f207e45efea50135a16f5fcbc99. 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b91248305f9bd170badfc1ac16c2f2b, disabling compactions & flushes 2024-11-13T18:19:03,557 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. after waiting 0 ms 2024-11-13T18:19:03,557 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:19:03,558 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1b91248305f9bd170badfc1ac16c2f2b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:03,596 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:19:03,619 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/.tmp/colfamily11/2a355714f58f47f6ac29c78605232f7b is 52, key is sss/colfamily11:colfamily11/1731521753403/Put/seqid=0 2024-11-13T18:19:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741902_1078 (size=39263) 2024-11-13T18:19:03,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741902_1078 (size=39263) 2024-11-13T18:19:03,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741902_1078 (size=39263) 2024-11-13T18:19:03,697 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/.tmp/colfamily11/2a355714f58f47f6ac29c78605232f7b 2024-11-13T18:19:03,729 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/.tmp/colfamily11/2a355714f58f47f6ac29c78605232f7b as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/colfamily11/2a355714f58f47f6ac29c78605232f7b 2024-11-13T18:19:03,756 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/colfamily11/2a355714f58f47f6ac29c78605232f7b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:03,758 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1b91248305f9bd170badfc1ac16c2f2b in 201ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:03,796 DEBUG [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1351): Waiting on 1b91248305f9bd170badfc1ac16c2f2b, 38c892b7353daa24a9b6c917f1aa0a89 2024-11-13T18:19:03,820 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1b91248305f9bd170badfc1ac16c2f2b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:03,824 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b91248305f9bd170badfc1ac16c2f2b: Waiting for close lock at 1731521943557Running coprocessor pre-close hooks at 1731521943557Disabling compacts and flushes for region at 1731521943557Disabling writes for close at 1731521943557Obtaining lock to block concurrent updates at 1731521943558 (+1 ms)Preparing flush snapshotting stores in 1b91248305f9bd170badfc1ac16c2f2b at 1731521943558Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521943558Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. at 1731521943559 (+1 ms)Flushing 1b91248305f9bd170badfc1ac16c2f2b/colfamily11: creating writer at 1731521943559Flushing 1b91248305f9bd170badfc1ac16c2f2b/colfamily11: appending metadata at 1731521943618 (+59 ms)Flushing 1b91248305f9bd170badfc1ac16c2f2b/colfamily11: closing flushed file at 1731521943618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74da5cd4: reopening flushed file at 1731521943726 (+108 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1b91248305f9bd170badfc1ac16c2f2b in 201ms, sequenceid=5, compaction requested=false at 1731521943758 (+32 ms)Writing region close event to WAL at 1731521943794 (+36 ms)Running coprocessor post-close hooks at 1731521943824 (+30 ms)Closed at 1731521943824 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731521748491.1b91248305f9bd170badfc1ac16c2f2b. 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38c892b7353daa24a9b6c917f1aa0a89, disabling compactions & flushes 2024-11-13T18:19:03,825 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. after waiting 0 ms 2024-11-13T18:19:03,825 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:19:03,825 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 38c892b7353daa24a9b6c917f1aa0a89 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T18:19:03,864 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/.tmp/colfamily11/4dbdd3ee1e22425eab6026b78bfed0ea is 52, key is bbb/colfamily11:colfamily11/1731521752899/Put/seqid=0 2024-11-13T18:19:03,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741903_1079 (size=39263) 2024-11-13T18:19:03,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741903_1079 (size=39263) 2024-11-13T18:19:03,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741903_1079 (size=39263) 2024-11-13T18:19:03,884 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/.tmp/colfamily11/4dbdd3ee1e22425eab6026b78bfed0ea 2024-11-13T18:19:03,903 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/.tmp/colfamily11/4dbdd3ee1e22425eab6026b78bfed0ea as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/colfamily11/4dbdd3ee1e22425eab6026b78bfed0ea 2024-11-13T18:19:03,910 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/colfamily11/4dbdd3ee1e22425eab6026b78bfed0ea, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T18:19:03,913 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 38c892b7353daa24a9b6c917f1aa0a89 in 88ms, sequenceid=5, compaction requested=false 2024-11-13T18:19:03,925 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/38c892b7353daa24a9b6c917f1aa0a89/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:19:03,926 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:19:03,926 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38c892b7353daa24a9b6c917f1aa0a89: Waiting for close lock at 1731521943825Running coprocessor pre-close hooks at 1731521943825Disabling compacts and flushes for region at 1731521943825Disabling writes for close at 1731521943825Obtaining lock to block concurrent updates at 1731521943825Preparing flush snapshotting stores in 38c892b7353daa24a9b6c917f1aa0a89 at 1731521943825Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731521943826 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. at 1731521943828 (+2 ms)Flushing 38c892b7353daa24a9b6c917f1aa0a89/colfamily11: creating writer at 1731521943828Flushing 38c892b7353daa24a9b6c917f1aa0a89/colfamily11: appending metadata at 1731521943863 (+35 ms)Flushing 38c892b7353daa24a9b6c917f1aa0a89/colfamily11: closing flushed file at 1731521943863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e338501: reopening flushed file at 1731521943902 (+39 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 38c892b7353daa24a9b6c917f1aa0a89 in 88ms, sequenceid=5, compaction requested=false at 1731521943913 (+11 ms)Writing region close event to WAL at 1731521943917 (+4 ms)Running coprocessor post-close hooks at 1731521943926 (+9 ms)Closed at 1731521943926 2024-11-13T18:19:03,926 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731521748491.38c892b7353daa24a9b6c917f1aa0a89. 2024-11-13T18:19:03,997 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,35435,1731521744991; all regions closed. 2024-11-13T18:19:03,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741865_1041 (size=14124) 2024-11-13T18:19:04,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741865_1041 (size=14124) 2024-11-13T18:19:04,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741865_1041 (size=14124) 2024-11-13T18:19:04,002 DEBUG [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs 2024-11-13T18:19:04,002 INFO [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C35435%2C1731521744991.meta:.meta(num 1731521934948) 2024-11-13T18:19:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741833_1009 (size=353044) 2024-11-13T18:19:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741833_1009 (size=353044) 2024-11-13T18:19:04,005 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/WALs/fda96cf025dc,35435,1731521744991/fda96cf025dc%2C35435%2C1731521744991.1731521747365 not finished, retry = 0 2024-11-13T18:19:04,109 DEBUG [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/oldWALs 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C35435%2C1731521744991:(num 1731521747365) 2024-11-13T18:19:04,109 DEBUG [RS:0;fda96cf025dc:35435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:19:04,109 INFO [RS:0;fda96cf025dc:35435 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35435 2024-11-13T18:19:04,111 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:19:04,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,35435,1731521744991 2024-11-13T18:19:04,112 INFO [RS:0;fda96cf025dc:35435 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:19:04,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:19:04,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,35435,1731521744991] 2024-11-13T18:19:04,116 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,35435,1731521744991 already deleted, retry=false 2024-11-13T18:19:04,116 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,35435,1731521744991 expired; onlineServers=0 2024-11-13T18:19:04,116 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda96cf025dc,41233,1731521744053' ***** 2024-11-13T18:19:04,116 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:19:04,117 DEBUG [M:0;fda96cf025dc:41233 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:19:04,117 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:19:04,117 DEBUG [M:0;fda96cf025dc:41233 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:19:04,117 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521746895 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731521746895,5,FailOnTimeoutGroup] 2024-11-13T18:19:04,117 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521746893 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731521746893,5,FailOnTimeoutGroup] 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] hbase.ChoreService(370): Chore service for: master/fda96cf025dc:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:19:04,117 DEBUG [M:0;fda96cf025dc:41233 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:19:04,117 INFO [M:0;fda96cf025dc:41233 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:19:04,118 INFO [M:0;fda96cf025dc:41233 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:19:04,118 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:19:04,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:19:04,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:19:04,119 DEBUG [M:0;fda96cf025dc:41233 {}] zookeeper.ZKUtil(347): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:19:04,120 WARN [M:0;fda96cf025dc:41233 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:19:04,120 INFO [M:0;fda96cf025dc:41233 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/.lastflushedseqids 2024-11-13T18:19:04,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741904_1080 (size=2864) 2024-11-13T18:19:04,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741904_1080 (size=2864) 2024-11-13T18:19:04,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741904_1080 (size=2864) 2024-11-13T18:19:04,176 INFO [M:0;fda96cf025dc:41233 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:19:04,177 INFO [M:0;fda96cf025dc:41233 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:19:04,177 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:19:04,177 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:04,177 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:04,177 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:19:04,177 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:04,177 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=335.62 KB heapSize=394.47 KB 2024-11-13T18:19:04,200 DEBUG [M:0;fda96cf025dc:41233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14ab9dd74e5b4825bc7cfff9ed764a5d is 82, key is hbase:meta,,1/info:regioninfo/1731521935335/Put/seqid=0 2024-11-13T18:19:04,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741905_1081 (size=6071) 2024-11-13T18:19:04,215 INFO [RS:0;fda96cf025dc:35435 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:19:04,215 INFO [RS:0;fda96cf025dc:35435 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,35435,1731521744991; zookeeper connection closed. 2024-11-13T18:19:04,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:04,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35435-0x1003a5b1ab20001, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:04,215 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3455ba51 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3455ba51 2024-11-13T18:19:04,216 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T18:19:04,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741905_1081 (size=6071) 2024-11-13T18:19:04,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741905_1081 (size=6071) 2024-11-13T18:19:04,220 INFO [M:0;fda96cf025dc:41233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14ab9dd74e5b4825bc7cfff9ed764a5d 2024-11-13T18:19:04,249 DEBUG [M:0;fda96cf025dc:41233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f80b89603d94641b425775979f8b738 is 4106, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731521750365/Put/seqid=0 2024-11-13T18:19:04,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741906_1082 (size=13327) 2024-11-13T18:19:04,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741906_1082 (size=13327) 2024-11-13T18:19:04,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741906_1082 (size=13327) 2024-11-13T18:19:04,257 INFO [M:0;fda96cf025dc:41233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=334.39 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f80b89603d94641b425775979f8b738 2024-11-13T18:19:04,277 DEBUG [M:0;fda96cf025dc:41233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0700f301b41345459880d1af53a3f351 is 69, key is fda96cf025dc,35435,1731521744991/rs:state/1731521746971/Put/seqid=0 2024-11-13T18:19:04,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34407 is added to blk_1073741907_1083 (size=5447) 2024-11-13T18:19:04,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741907_1083 (size=5447) 2024-11-13T18:19:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741907_1083 (size=5447) 2024-11-13T18:19:04,290 INFO [M:0;fda96cf025dc:41233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0700f301b41345459880d1af53a3f351 2024-11-13T18:19:04,295 INFO [M:0;fda96cf025dc:41233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0700f301b41345459880d1af53a3f351 2024-11-13T18:19:04,295 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14ab9dd74e5b4825bc7cfff9ed764a5d as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/14ab9dd74e5b4825bc7cfff9ed764a5d 2024-11-13T18:19:04,300 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/14ab9dd74e5b4825bc7cfff9ed764a5d, entries=14, sequenceid=618, filesize=5.9 K 2024-11-13T18:19:04,301 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f80b89603d94641b425775979f8b738 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f80b89603d94641b425775979f8b738 2024-11-13T18:19:04,306 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f80b89603d94641b425775979f8b738, entries=79, sequenceid=618, filesize=13.0 K 2024-11-13T18:19:04,307 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0700f301b41345459880d1af53a3f351 as hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0700f301b41345459880d1af53a3f351 2024-11-13T18:19:04,311 INFO [M:0;fda96cf025dc:41233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0700f301b41345459880d1af53a3f351 2024-11-13T18:19:04,311 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35347/user/jenkins/test-data/bf7e776a-289f-9ab1-5905-ba1ac9f70ada/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0700f301b41345459880d1af53a3f351, entries=3, sequenceid=618, filesize=5.3 K 2024-11-13T18:19:04,312 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(3140): Finished flush of dataSize ~335.62 KB/343670, heapSize ~394.17 KB/403632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=618, compaction requested=false 2024-11-13T18:19:04,313 INFO [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:19:04,313 DEBUG [M:0;fda96cf025dc:41233 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731521944177Disabling compacts and flushes for region at 1731521944177Disabling writes for close at 1731521944177Obtaining lock to block concurrent updates at 1731521944177Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731521944177Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=343670, getHeapSize=403872, getOffHeapSize=0, getCellsCount=779 at 1731521944178 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731521944178Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731521944178Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731521944199 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731521944199Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731521944227 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731521944249 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731521944249Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731521944262 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731521944277 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731521944277Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32b65141: reopening flushed file at 1731521944295 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15644192: reopening flushed file at 1731521944300 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76e5b373: reopening flushed file at 1731521944306 (+6 ms)Finished flush of dataSize ~335.62 KB/343670, heapSize ~394.17 KB/403632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=618, compaction requested=false at 1731521944312 (+6 ms)Writing region close event to WAL at 1731521944313 (+1 ms)Closed at 1731521944313 2024-11-13T18:19:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37435 is added to blk_1073741830_1006 (size=384903) 2024-11-13T18:19:04,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38349 is added to blk_1073741830_1006 (size=384903) 2024-11-13T18:19:04,316 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:19:04,317 INFO [M:0;fda96cf025dc:41233 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:19:04,317 INFO [M:0;fda96cf025dc:41233 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41233 2024-11-13T18:19:04,317 INFO [M:0;fda96cf025dc:41233 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:19:04,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:04,422 INFO [M:0;fda96cf025dc:41233 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:19:04,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41233-0x1003a5b1ab20000, quorum=127.0.0.1:55845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:19:04,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10fe3f07{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:19:04,438 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5904946f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:19:04,438 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:19:04,439 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ebf56ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:19:04,439 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f411fc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,STOPPED} 2024-11-13T18:19:04,442 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:19:04,442 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1886344949-172.17.0.3-1731521739526 (Datanode Uuid 10001456-81e1-4ad3-9670-f5c1bebc8b50) service to localhost/127.0.0.1:35347 2024-11-13T18:19:04,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data5/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data6/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,445 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:19:04,445 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:19:04,446 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:19:04,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2925d9ef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:19:04,454 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74214a4f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:19:04,454 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:19:04,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5480bc23{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:19:04,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b86904d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,STOPPED} 2024-11-13T18:19:04,455 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:19:04,455 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:19:04,455 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1886344949-172.17.0.3-1731521739526 (Datanode Uuid 54b869a2-0ee7-4070-8387-5049c9b58854) service to localhost/127.0.0.1:35347 2024-11-13T18:19:04,455 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:19:04,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data3/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data4/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,456 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:19:04,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7937b685{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:19:04,465 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15e7d6d5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:19:04,465 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:19:04,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71f8e357{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:19:04,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ee08c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,STOPPED} 2024-11-13T18:19:04,466 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:19:04,466 WARN [BP-1886344949-172.17.0.3-1731521739526 heartbeating to localhost/127.0.0.1:35347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1886344949-172.17.0.3-1731521739526 (Datanode Uuid fcba98fd-d1a7-412b-b731-bd9fcad4325a) service to localhost/127.0.0.1:35347 2024-11-13T18:19:04,466 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:19:04,466 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:19:04,467 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data1/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,467 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/cluster_89d13e0c-d1ea-0273-6bed-85ffcaad47d5/data/data2/current/BP-1886344949-172.17.0.3-1731521739526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:19:04,467 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:19:04,476 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@681e6b33{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:19:04,477 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@66ea3674{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:19:04,477 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:19:04,477 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3db55f04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:19:04,477 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@667e8272{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bb0cf3d6-b34f-476f-25b2-498c3d5708d5/hadoop.log.dir/,STOPPED} 2024-11-13T18:19:04,494 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:19:04,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T18:19:04,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=93 (was 11) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/fda96cf025dc:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (641675642) connection to localhost/127.0.0.1:35347 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35347 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/fda96cf025dc:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fda96cf025dc:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fda96cf025dc:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35347 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (641675642) connection to localhost/127.0.0.1:35347 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fda96cf025dc:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35347 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fda96cf025dc:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35347 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35347 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (641675642) connection to localhost/127.0.0.1:35347 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35347 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fda96cf025dc:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (641675642) connection to localhost/127.0.0.1:35347 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35347 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@29b7a5ea java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=830 (was 691) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1532 (was 2106)