2024-11-08 00:43:37,377 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-08 00:43:37,388 main DEBUG Took 0.009103 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-08 00:43:37,389 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-08 00:43:37,389 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-08 00:43:37,390 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-08 00:43:37,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,397 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-08 00:43:37,410 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,412 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,412 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,414 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,414 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,415 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,416 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,417 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,418 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,419 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,419 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,420 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 00:43:37,421 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,421 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-08 00:43:37,423 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 00:43:37,424 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-08 00:43:37,426 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-08 00:43:37,427 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-08 00:43:37,428 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-08 00:43:37,429 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-08 00:43:37,441 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-08 00:43:37,444 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-08 00:43:37,446 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-08 00:43:37,447 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-08 00:43:37,448 main DEBUG createAppenders(={Console})
2024-11-08 00:43:37,449 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized
2024-11-08 00:43:37,449 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-08 00:43:37,449 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK.
2024-11-08 00:43:37,450 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-08 00:43:37,450 main DEBUG OutputStream closed
2024-11-08 00:43:37,451 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-08 00:43:37,451 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-08 00:43:37,451 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK
2024-11-08 00:43:37,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-08 00:43:37,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-08 00:43:37,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-08 00:43:37,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-08 00:43:37,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-08 00:43:37,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-08 00:43:37,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-08 00:43:37,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-08 00:43:37,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-08 00:43:37,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-08 00:43:37,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-08 00:43:37,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-08 00:43:37,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-08 00:43:37,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-08 00:43:37,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-08 00:43:37,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-08 00:43:37,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-08 00:43:37,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-08 00:43:37,540 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08 00:43:37,541 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null
2024-11-08 00:43:37,541 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-08 00:43:37,542 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK.
2024-11-08T00:43:37,768 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90
2024-11-08 00:43:37,771 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-08 00:43:37,771 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08T00:43:37,780 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins
2024-11-08T00:43:37,799 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-08T00:43:37,813 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e, deleteOnExit=true
2024-11-08T00:43:37,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-08T00:43:37,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/test.cache.data in system properties and HBase conf
2024-11-08T00:43:37,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.tmp.dir in system properties and HBase conf
2024-11-08T00:43:37,816 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.log.dir in system properties and HBase conf
2024-11-08T00:43:37,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-08T00:43:37,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-08T00:43:37,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-08T00:43:37,919 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-08T00:43:38,011 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-08T00:43:38,015 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-08T00:43:38,015 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-08T00:43:38,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-08T00:43:38,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T00:43:38,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-08T00:43:38,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-08T00:43:38,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T00:43:38,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T00:43:38,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-08T00:43:38,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/nfs.dump.dir in system properties and HBase conf
2024-11-08T00:43:38,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/java.io.tmpdir in system properties and HBase conf
2024-11-08T00:43:38,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T00:43:38,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-08T00:43:38,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-08T00:43:38,824 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-08T00:43:38,909 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2230ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-08T00:43:38,989 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T00:43:39,059 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T00:43:39,080 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T00:43:39,080 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T00:43:39,081 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-08T00:43:39,094 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T00:43:39,096 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.log.dir/,AVAILABLE}
2024-11-08T00:43:39,097 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T00:43:39,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27b19a13{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/java.io.tmpdir/jetty-localhost-33831-hadoop-hdfs-3_4_1-tests_jar-_-any-10412382766515298271/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T00:43:39,302 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:33831}
2024-11-08T00:43:39,303 INFO  [Time-limited test {}] server.Server(415): Started @2625ms
2024-11-08T00:43:39,704 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T00:43:39,711 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T00:43:39,712 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T00:43:39,712 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T00:43:39,712 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T00:43:39,713 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.log.dir/,AVAILABLE}
2024-11-08T00:43:39,713 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T00:43:39,832 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156ffeaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/java.io.tmpdir/jetty-localhost-40821-hadoop-hdfs-3_4_1-tests_jar-_-any-12478830480075705136/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T00:43:39,833 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:40821}
2024-11-08T00:43:39,833 INFO  [Time-limited test {}] server.Server(415): Started @3156ms
2024-11-08T00:43:39,888 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T00:43:40,356 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/data/data1/current/BP-1589806164-172.17.0.2-1731026618582/current, will proceed with Du for space computation calculation, 
2024-11-08T00:43:40,356 WARN  [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/data/data2/current/BP-1589806164-172.17.0.2-1731026618582/current, will proceed with Du for space computation calculation, 
2024-11-08T00:43:40,398 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T00:43:40,454 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c44efbe16187eb with lease ID 0x7fef6a4419d8db99: Processing first storage report for DS-cf77c358-92c0-4133-b12c-8b96a18c0ba6 from datanode DatanodeRegistration(127.0.0.1:44031, datanodeUuid=48d57e7a-9ae2-45e9-b239-10bcdf31a27b, infoPort=40545, infoSecurePort=0, ipcPort=34897, storageInfo=lv=-57;cid=testClusterID;nsid=1365985579;c=1731026618582)
2024-11-08T00:43:40,455 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c44efbe16187eb with lease ID 0x7fef6a4419d8db99: from storage DS-cf77c358-92c0-4133-b12c-8b96a18c0ba6 node DatanodeRegistration(127.0.0.1:44031, datanodeUuid=48d57e7a-9ae2-45e9-b239-10bcdf31a27b, infoPort=40545, infoSecurePort=0, ipcPort=34897, storageInfo=lv=-57;cid=testClusterID;nsid=1365985579;c=1731026618582), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-08T00:43:40,456 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c44efbe16187eb with lease ID 0x7fef6a4419d8db99: Processing first storage report for DS-65ea3591-b647-4252-b7dc-3e187b87dff3 from datanode DatanodeRegistration(127.0.0.1:44031, datanodeUuid=48d57e7a-9ae2-45e9-b239-10bcdf31a27b, infoPort=40545, infoSecurePort=0, ipcPort=34897, storageInfo=lv=-57;cid=testClusterID;nsid=1365985579;c=1731026618582)
2024-11-08T00:43:40,456 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c44efbe16187eb with lease ID 0x7fef6a4419d8db99: from storage DS-65ea3591-b647-4252-b7dc-3e187b87dff3 node DatanodeRegistration(127.0.0.1:44031, datanodeUuid=48d57e7a-9ae2-45e9-b239-10bcdf31a27b, infoPort=40545, infoSecurePort=0, ipcPort=34897, storageInfo=lv=-57;cid=testClusterID;nsid=1365985579;c=1731026618582), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T00:43:40,542 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90
2024-11-08T00:43:40,619 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/zookeeper_0, clientPort=53666, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-08T00:43:40,629 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53666
2024-11-08T00:43:40,644 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:40,650 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:40,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741825_1001 (size=7)
2024-11-08T00:43:41,307 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269 with version=8
2024-11-08T00:43:41,307 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/hbase-staging
2024-11-08T00:43:41,411 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-08T00:43:41,666 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c302dddf724e:0 server-side Connection retries=45
2024-11-08T00:43:41,677 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:41,677 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:41,682 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T00:43:41,682 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:41,682 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T00:43:41,820 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-08T00:43:41,881 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-08T00:43:41,889 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-08T00:43:41,893 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T00:43:41,920 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6565 (auto-detected)
2024-11-08T00:43:41,921 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-08T00:43:41,939 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44077
2024-11-08T00:43:41,961 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44077 connecting to ZooKeeper ensemble=127.0.0.1:53666
2024-11-08T00:43:41,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:440770x0, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T00:43:41,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44077-0x100f507fb490000 connected
2024-11-08T00:43:42,021 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:42,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:42,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T00:43:42,040 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269, hbase.cluster.distributed=false
2024-11-08T00:43:42,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T00:43:42,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44077
2024-11-08T00:43:42,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44077
2024-11-08T00:43:42,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44077
2024-11-08T00:43:42,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44077
2024-11-08T00:43:42,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44077
2024-11-08T00:43:42,225 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c302dddf724e:0 server-side Connection retries=45
2024-11-08T00:43:42,227 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:42,227 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:42,228 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T00:43:42,228 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T00:43:42,228 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T00:43:42,231 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T00:43:42,234 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T00:43:42,235 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46797
2024-11-08T00:43:42,237 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46797 connecting to ZooKeeper ensemble=127.0.0.1:53666
2024-11-08T00:43:42,239 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:42,243 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:42,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467970x0, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T00:43:42,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467970x0, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T00:43:42,259 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T00:43:42,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46797-0x100f507fb490001 connected
2024-11-08T00:43:42,269 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T00:43:42,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T00:43:42,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T00:43:42,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46797
2024-11-08T00:43:42,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46797
2024-11-08T00:43:42,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46797
2024-11-08T00:43:42,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46797
2024-11-08T00:43:42,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46797
2024-11-08T00:43:42,308 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c302dddf724e:44077
2024-11-08T00:43:42,309 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c302dddf724e,44077,1731026621470
2024-11-08T00:43:42,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T00:43:42,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T00:43:42,319 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c302dddf724e,44077,1731026621470
2024-11-08T00:43:42,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T00:43:42,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:42,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:42,342 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-08T00:43:42,344 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c302dddf724e,44077,1731026621470 from backup master directory
2024-11-08T00:43:42,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c302dddf724e,44077,1731026621470
2024-11-08T00:43:42,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T00:43:42,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T00:43:42,347 WARN  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T00:43:42,347 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c302dddf724e,44077,1731026621470
2024-11-08T00:43:42,350 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-08T00:43:42,351 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-08T00:43:42,409 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/hbase.id] with ID: 3f512acf-79a1-47fd-a1a5-d47e1cd779db
2024-11-08T00:43:42,409 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/.tmp/hbase.id
2024-11-08T00:43:42,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741826_1002 (size=42)
2024-11-08T00:43:42,822 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/.tmp/hbase.id]:[hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/hbase.id]
2024-11-08T00:43:42,865 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T00:43:42,870 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-08T00:43:42,889 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-08T00:43:42,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:42,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:42,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741827_1003 (size=196)
2024-11-08T00:43:43,326 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-08T00:43:43,328 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-08T00:43:43,345 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:43,350 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T00:43:43,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741828_1004 (size=1189)
2024-11-08T00:43:43,800 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store
2024-11-08T00:43:43,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741829_1005 (size=34)
2024-11-08T00:43:44,224 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-08T00:43:44,227 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:44,228 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T00:43:44,229 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:44,229 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:44,230 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T00:43:44,230 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:44,230 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:44,231 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731026624228Disabling compacts and flushes for region at 1731026624228Disabling writes for close at 1731026624230 (+2 ms)Writing region close event to WAL at 1731026624230Closed at 1731026624230
2024-11-08T00:43:44,234 WARN  [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/.initializing
2024-11-08T00:43:44,234 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/WALs/c302dddf724e,44077,1731026621470
2024-11-08T00:43:44,242 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-08T00:43:44,257 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c302dddf724e%2C44077%2C1731026621470, suffix=, logDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/WALs/c302dddf724e,44077,1731026621470, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/oldWALs, maxLogs=10
2024-11-08T00:43:44,281 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/WALs/c302dddf724e,44077,1731026621470/c302dddf724e%2C44077%2C1731026621470.1731026624262, exclude list is [], retry=0
2024-11-08T00:43:44,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44031,DS-cf77c358-92c0-4133-b12c-8b96a18c0ba6,DISK]
2024-11-08T00:43:44,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-11-08T00:43:44,339 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/WALs/c302dddf724e,44077,1731026621470/c302dddf724e%2C44077%2C1731026621470.1731026624262
2024-11-08T00:43:44,340 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40545:40545)]
2024-11-08T00:43:44,340 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-08T00:43:44,341 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:44,344 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,345 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,383 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T00:43:44,408 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-08T00:43:44,412 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,414 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:44,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T00:43:44,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-08T00:43:44,418 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:44,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T00:43:44,422 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-08T00:43:44,422 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:44,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T00:43:44,425 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-08T00:43:44,425 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,426 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:44,427 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,430 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,431 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,436 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,437 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,440 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T00:43:44,444 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T00:43:44,448 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:44,449 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65934020, jitterRate=-0.017506539821624756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T00:43:44,455 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731026624358Initializing all the Stores at 1731026624360 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026624361 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026624361Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026624362 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026624362Cleaning up temporary data from old regions at 1731026624437 (+75 ms)Region opened successfully at 1731026624455 (+18 ms)
2024-11-08T00:43:44,456 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-08T00:43:44,489 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753bbbdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0
2024-11-08T00:43:44,520 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-08T00:43:44,532 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:44,533 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:44,536 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:44,537 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:44,543 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:44,543 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-08T00:43:44,569 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-08T00:43:44,577 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-08T00:43:44,580 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-08T00:43:44,582 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-08T00:43:44,584 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-08T00:43:44,585 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-08T00:43:44,587 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-08T00:43:44,591 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-08T00:43:44,592 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-08T00:43:44,593 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-08T00:43:44,596 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-08T00:43:44,614 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-08T00:43:44,616 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-08T00:43:44,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T00:43:44,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T00:43:44,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,623 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c302dddf724e,44077,1731026621470, sessionid=0x100f507fb490000, setting cluster-up flag (Was=false)
2024-11-08T00:43:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,644 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-08T00:43:44,646 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,44077,1731026621470
2024-11-08T00:43:44,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:44,660 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-08T00:43:44,662 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c302dddf724e,44077,1731026621470
2024-11-08T00:43:44,669 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-08T00:43:44,695 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(746): ClusterId : 3f512acf-79a1-47fd-a1a5-d47e1cd779db
2024-11-08T00:43:44,698 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-08T00:43:44,703 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-08T00:43:44,703 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-08T00:43:44,706 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-08T00:43:44,706 DEBUG [RS:0;c302dddf724e:46797 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168fe345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c302dddf724e/172.17.0.2:0
2024-11-08T00:43:44,731 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c302dddf724e:46797
2024-11-08T00:43:44,735 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-08T00:43:44,735 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-08T00:43:44,735 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-08T00:43:44,738 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,44077,1731026621470 with port=46797, startcode=1731026622161
2024-11-08T00:43:44,748 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:44,753 DEBUG [RS:0;c302dddf724e:46797 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-08T00:43:44,761 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-08T00:43:44,769 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-08T00:43:44,775 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c302dddf724e,44077,1731026621470
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-08T00:43:44,783 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5
2024-11-08T00:43:44,783 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5
2024-11-08T00:43:44,783 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5
2024-11-08T00:43:44,783 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c302dddf724e:0, corePoolSize=5, maxPoolSize=5
2024-11-08T00:43:44,784 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c302dddf724e:0, corePoolSize=10, maxPoolSize=10
2024-11-08T00:43:44,784 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:44,784 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c302dddf724e:0, corePoolSize=2, maxPoolSize=2
2024-11-08T00:43:44,784 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:44,786 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026654786
2024-11-08T00:43:44,788 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-08T00:43:44,789 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-08T00:43:44,790 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:44,791 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-08T00:43:44,794 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-08T00:43:44,794 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-08T00:43:44,795 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-08T00:43:44,795 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-08T00:43:44,795 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-08T00:43:44,796 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:44,798 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,799 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-08T00:43:44,800 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-08T00:43:44,802 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-08T00:43:44,802 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:44,802 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-08T00:43:44,807 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-08T00:43:44,807 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-08T00:43:44,810 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026624809,5,FailOnTimeoutGroup]
2024-11-08T00:43:44,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741831_1007 (size=1321)
2024-11-08T00:43:44,811 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026624810,5,FailOnTimeoutGroup]
2024-11-08T00:43:44,811 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:44,811 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-08T00:43:44,812 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:44,812 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-08T00:43:44,812 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:44,813 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:44,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741832_1008 (size=32)
2024-11-08T00:43:44,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:44,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T00:43:44,832 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-11-08T00:43:44,832 WARN  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-11-08T00:43:44,834 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T00:43:44,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,835 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:44,836 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T00:43:44,839 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T00:43:44,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,840 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:44,841 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T00:43:44,844 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T00:43:44,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,845 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:44,846 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T00:43:44,849 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T00:43:44,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:44,850 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:44,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T00:43:44,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740
2024-11-08T00:43:44,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740
2024-11-08T00:43:44,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T00:43:44,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T00:43:44,857 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T00:43:44,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T00:43:44,864 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:44,865 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66197180, jitterRate=-0.013585150241851807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T00:43:44,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731026624829Initializing all the Stores at 1731026624831 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026624831Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026624831Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026624831Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026624831Cleaning up temporary data from old regions at 1731026624856 (+25 ms)Region opened successfully at 1731026624867 (+11 ms)
2024-11-08T00:43:44,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T00:43:44,868 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T00:43:44,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T00:43:44,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T00:43:44,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T00:43:44,870 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T00:43:44,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731026624868Disabling compacts and flushes for region at 1731026624868Disabling writes for close at 1731026624868Writing region close event to WAL at 1731026624869 (+1 ms)Closed at 1731026624870 (+1 ms)
2024-11-08T00:43:44,874 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:44,874 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-08T00:43:44,880 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-08T00:43:44,891 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T00:43:44,893 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-08T00:43:44,934 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(2659): reportForDuty to master=c302dddf724e,44077,1731026621470 with port=46797, startcode=1731026622161
2024-11-08T00:43:44,936 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c302dddf724e,46797,1731026622161
2024-11-08T00:43:44,939 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] master.ServerManager(517): Registering regionserver=c302dddf724e,46797,1731026622161
2024-11-08T00:43:44,948 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:44,948 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44739
2024-11-08T00:43:44,948 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-08T00:43:44,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T00:43:44,953 DEBUG [RS:0;c302dddf724e:46797 {}] zookeeper.ZKUtil(111): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c302dddf724e,46797,1731026622161
2024-11-08T00:43:44,954 WARN  [RS:0;c302dddf724e:46797 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T00:43:44,954 INFO  [RS:0;c302dddf724e:46797 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T00:43:44,954 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161
2024-11-08T00:43:44,956 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c302dddf724e,46797,1731026622161]
2024-11-08T00:43:44,981 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-08T00:43:44,994 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-08T00:43:44,999 INFO  [RS:0;c302dddf724e:46797 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-08T00:43:44,999 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,000 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-08T00:43:45,005 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-08T00:43:45,007 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,007 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,007 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c302dddf724e:0, corePoolSize=2, maxPoolSize=2
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,008 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c302dddf724e:0, corePoolSize=1, maxPoolSize=1
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3
2024-11-08T00:43:45,009 DEBUG [RS:0;c302dddf724e:46797 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c302dddf724e:0, corePoolSize=3, maxPoolSize=3
2024-11-08T00:43:45,010 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,010 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,011 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,011 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,011 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,011 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,46797,1731026622161-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T00:43:45,032 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-08T00:43:45,034 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,46797,1731026622161-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,035 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,035 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.Replication(171): c302dddf724e,46797,1731026622161 started
2024-11-08T00:43:45,045 WARN  [c302dddf724e:44077 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-08T00:43:45,055 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,055 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1482): Serving as c302dddf724e,46797,1731026622161, RpcServer on c302dddf724e/172.17.0.2:46797, sessionid=0x100f507fb490001
2024-11-08T00:43:45,056 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-08T00:43:45,056 DEBUG [RS:0;c302dddf724e:46797 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c302dddf724e,46797,1731026622161
2024-11-08T00:43:45,056 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,46797,1731026622161'
2024-11-08T00:43:45,057 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-08T00:43:45,058 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-08T00:43:45,058 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-08T00:43:45,058 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-08T00:43:45,059 DEBUG [RS:0;c302dddf724e:46797 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c302dddf724e,46797,1731026622161
2024-11-08T00:43:45,059 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c302dddf724e,46797,1731026622161'
2024-11-08T00:43:45,059 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-08T00:43:45,060 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-08T00:43:45,060 DEBUG [RS:0;c302dddf724e:46797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-08T00:43:45,060 INFO  [RS:0;c302dddf724e:46797 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-08T00:43:45,061 INFO  [RS:0;c302dddf724e:46797 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-08T00:43:45,166 INFO  [RS:0;c302dddf724e:46797 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-08T00:43:45,170 INFO  [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c302dddf724e%2C46797%2C1731026622161, suffix=, logDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/oldWALs, maxLogs=32
2024-11-08T00:43:45,186 DEBUG [RS:0;c302dddf724e:46797 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161/c302dddf724e%2C46797%2C1731026622161.1731026625172, exclude list is [], retry=0
2024-11-08T00:43:45,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44031,DS-cf77c358-92c0-4133-b12c-8b96a18c0ba6,DISK]
2024-11-08T00:43:45,194 INFO  [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161/c302dddf724e%2C46797%2C1731026622161.1731026625172
2024-11-08T00:43:45,195 DEBUG [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40545:40545)]
2024-11-08T00:43:45,297 DEBUG [c302dddf724e:44077 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-08T00:43:45,310 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:45,317 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPENING
2024-11-08T00:43:45,323 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-08T00:43:45,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:45,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:45,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:45,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:45,327 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T00:43:45,329 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:45,507 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-08T00:43:45,511 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52347, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-08T00:43:45,524 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-08T00:43:45,524 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T00:43:45,525 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-11-08T00:43:45,529 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c302dddf724e%2C46797%2C1731026622161.meta, suffix=.meta, logDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/oldWALs, maxLogs=32
2024-11-08T00:43:45,554 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161/c302dddf724e%2C46797%2C1731026622161.meta.1731026625532.meta, exclude list is [], retry=0
2024-11-08T00:43:45,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44031,DS-cf77c358-92c0-4133-b12c-8b96a18c0ba6,DISK]
2024-11-08T00:43:45,562 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/WALs/c302dddf724e,46797,1731026622161/c302dddf724e%2C46797%2C1731026622161.meta.1731026625532.meta
2024-11-08T00:43:45,562 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40545:40545)]
2024-11-08T00:43:45,562 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-08T00:43:45,564 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-08T00:43:45,567 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-08T00:43:45,572 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-08T00:43:45,576 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-08T00:43:45,576 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:45,577 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-08T00:43:45,577 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-08T00:43:45,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T00:43:45,581 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T00:43:45,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:45,582 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:45,582 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T00:43:45,584 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T00:43:45,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:45,585 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:45,585 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T00:43:45,586 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T00:43:45,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:45,587 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:45,587 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T00:43:45,588 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T00:43:45,588 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:45,589 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T00:43:45,589 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T00:43:45,590 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740
2024-11-08T00:43:45,593 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740
2024-11-08T00:43:45,595 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T00:43:45,595 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T00:43:45,596 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T00:43:45,598 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T00:43:45,600 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59875712, jitterRate=-0.10778236389160156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T00:43:45,600 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-08T00:43:45,602 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731026625577Writing region info on filesystem at 1731026625577Initializing all the Stores at 1731026625579 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026625579Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026625580 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026625580Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731026625580Cleaning up temporary data from old regions at 1731026625595 (+15 ms)Running coprocessor post-open hooks at 1731026625600 (+5 ms)Region opened successfully at 1731026625601 (+1 ms)
2024-11-08T00:43:45,609 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731026625498
2024-11-08T00:43:45,620 DEBUG [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-08T00:43:45,621 INFO  [RS_OPEN_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-08T00:43:45,623 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:45,625 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:45,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:45,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:45,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:45,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:45,636 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:45,642 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-08T00:43:45,642 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c302dddf724e,46797,1731026622161 in 307 msec
2024-11-08T00:43:45,649 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-08T00:43:45,649 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 764 msec
2024-11-08T00:43:45,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:45,651 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-08T00:43:45,672 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T00:43:45,673 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,46797,1731026622161, seqNum=-1]
2024-11-08T00:43:45,693 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T00:43:45,696 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59233, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T00:43:45,714 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0170 sec
2024-11-08T00:43:45,715 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731026625714, completionTime=-1
2024-11-08T00:43:45,717 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-08T00:43:45,717 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:45,748 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:45,748 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026685748
2024-11-08T00:43:45,748 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026745748
2024-11-08T00:43:45,748 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec
2024-11-08T00:43:45,750 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,751 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,751 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,752 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c302dddf724e:44077, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,753 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,753 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,759 DEBUG [master/c302dddf724e:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-08T00:43:45,782 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.434sec
2024-11-08T00:43:45,783 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-08T00:43:45,784 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-08T00:43:45,785 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-08T00:43:45,786 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-08T00:43:45,786 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-08T00:43:45,787 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T00:43:45,787 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-08T00:43:45,795 DEBUG [master/c302dddf724e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-08T00:43:45,796 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-08T00:43:45,796 INFO  [master/c302dddf724e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c302dddf724e,44077,1731026621470-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T00:43:45,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371ae7bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T00:43:45,807 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-08T00:43:45,807 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-08T00:43:45,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c302dddf724e,44077,-1 for getting cluster id
2024-11-08T00:43:45,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-08T00:43:45,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3f512acf-79a1-47fd-a1a5-d47e1cd779db'
2024-11-08T00:43:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-08T00:43:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3f512acf-79a1-47fd-a1a5-d47e1cd779db"

2024-11-08T00:43:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8a2b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T00:43:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c302dddf724e,44077,-1]
2024-11-08T00:43:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-08T00:43:45,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:45,830 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-08T00:43:45,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9637c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T00:43:45,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T00:43:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c302dddf724e,46797,1731026622161, seqNum=-1]
2024-11-08T00:43:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T00:43:45,844 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T00:43:45,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c302dddf724e,44077,1731026621470
2024-11-08T00:43:45,878 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=207, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=2341
2024-11-08T00:43:45,881 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:45,907 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:45,911 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T00:43:45,914 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:45,916 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T00:43:45,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741835_1011 (size=639)
2024-11-08T00:43:46,336 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74ab50beefe6927e6e9d8ce049d56ee5, NAME => 'testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:46,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741836_1012 (size=51)
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 74ab50beefe6927e6e9d8ce049d56ee5, disabling compactions & flushes
2024-11-08T00:43:46,746 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5. after waiting 0 ms
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:46,746 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:46,746 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74ab50beefe6927e6e9d8ce049d56ee5:
Waiting for close lock at 1731026626746Disabling compacts and flushes for region at 1731026626746Disabling writes for close at 1731026626746Writing region close event to WAL at 1731026626746Closed at 1731026626746
2024-11-08T00:43:46,749 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T00:43:46,754 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731026626749"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026626749"}]},"ts":"1731026626749"}
2024-11-08T00:43:46,759 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-08T00:43:46,760 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T00:43:46,763 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026626761"}]},"ts":"1731026626761"}
2024-11-08T00:43:46,767 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta
2024-11-08T00:43:46,769 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, ASSIGN}]
2024-11-08T00:43:46,773 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, ASSIGN
2024-11-08T00:43:46,774 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:46,926 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=74ab50beefe6927e6e9d8ce049d56ee5, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:46,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, ASSIGN because future has completed
2024-11-08T00:43:46,932 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:47,092 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 74ab50beefe6927e6e9d8ce049d56ee5, NAME => 'testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.', STARTKEY => '', ENDKEY => ''}
2024-11-08T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,096 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 74ab50beefe6927e6e9d8ce049d56ee5 
2024-11-08T00:43:47,098 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74ab50beefe6927e6e9d8ce049d56ee5 columnFamilyName f1
2024-11-08T00:43:47,098 DEBUG [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:47,099 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] regionserver.HStore(327): Store=74ab50beefe6927e6e9d8ce049d56ee5/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:47,099 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 74ab50beefe6927e6e9d8ce049d56ee5 
2024-11-08T00:43:47,101 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74ab50beefe6927e6e9d8ce049d56ee5 columnFamilyName f2
2024-11-08T00:43:47,101 DEBUG [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:47,102 INFO  [StoreOpener-74ab50beefe6927e6e9d8ce049d56ee5-1 {}] regionserver.HStore(327): Store=74ab50beefe6927e6e9d8ce049d56ee5/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:47,102 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,103 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,104 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,105 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,105 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,106 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:47,108 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,111 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:47,112 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 74ab50beefe6927e6e9d8ce049d56ee5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66000603, jitterRate=-0.016514375805854797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:47,112 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,113 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 74ab50beefe6927e6e9d8ce049d56ee5:
Running coprocessor pre-open hook at 1731026627094Writing region info on filesystem at 1731026627094Initializing all the Stores at 1731026627095 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026627095Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026627095Cleaning up temporary data from old regions at 1731026627105 (+10 ms)Running coprocessor post-open hooks at 1731026627112 (+7 ms)Region opened successfully at 1731026627113 (+1 ms)
2024-11-08T00:43:47,115 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5., pid=6, masterSystemTime=1731026627085
2024-11-08T00:43:47,119 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,119 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,120 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=74ab50beefe6927e6e9d8ce049d56ee5, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:47,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:47,132 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-08T00:43:47,132 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161 in 195 msec
2024-11-08T00:43:47,138 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-08T00:43:47,138 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, ASSIGN in 363 msec
2024-11-08T00:43:47,140 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T00:43:47,140 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026627140"}]},"ts":"1731026627140"}
2024-11-08T00:43:47,144 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta
2024-11-08T00:43:47,146 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T00:43:47,150 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.2440 sec
2024-11-08T00:43:47,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:47,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026627172"}]},"ts":"1731026627172"}
2024-11-08T00:43:47,176 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta
2024-11-08T00:43:47,176 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING
2024-11-08T00:43:47,179 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}]
2024-11-08T00:43:47,188 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, UNASSIGN}]
2024-11-08T00:43:47,191 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, UNASSIGN
2024-11-08T00:43:47,192 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=74ab50beefe6927e6e9d8ce049d56ee5, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:47,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, UNASSIGN because future has completed
2024-11-08T00:43:47,197 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:47,197 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:47,356 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,357 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:47,357 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 74ab50beefe6927e6e9d8ce049d56ee5, disabling compactions & flushes
2024-11-08T00:43:47,357 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,357 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,358 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5. after waiting 0 ms
2024-11-08T00:43:47,358 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,364 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:47,367 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.
2024-11-08T00:43:47,367 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 74ab50beefe6927e6e9d8ce049d56ee5:
Waiting for close lock at 1731026627357Running coprocessor pre-close hooks at 1731026627357Disabling compacts and flushes for region at 1731026627357Disabling writes for close at 1731026627358 (+1 ms)Writing region close event to WAL at 1731026627358Running coprocessor post-close hooks at 1731026627365 (+7 ms)Closed at 1731026627367 (+2 ms)
2024-11-08T00:43:47,371 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,372 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=74ab50beefe6927e6e9d8ce049d56ee5, regionState=CLOSED
2024-11-08T00:43:47,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:47,382 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-08T00:43:47,382 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 74ab50beefe6927e6e9d8ce049d56ee5, server=c302dddf724e,46797,1731026622161 in 181 msec
2024-11-08T00:43:47,386 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-08T00:43:47,386 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=74ab50beefe6927e6e9d8ce049d56ee5, UNASSIGN in 195 msec
2024-11-08T00:43:47,391 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-08T00:43:47,391 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 209 msec
2024-11-08T00:43:47,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026627394"}]},"ts":"1731026627394"}
2024-11-08T00:43:47,397 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta
2024-11-08T00:43:47,397 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED
2024-11-08T00:43:47,401 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 237 msec
2024-11-08T00:43:47,418 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:47,422 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-08T00:43:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c302dddf724e,44077,1731026621470
2024-11-08T00:43:47,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5fcdfb41
2024-11-08T00:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-08T00:43:47,435 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-08T00:43:47,438 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable
2024-11-08T00:43:47,444 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTable
2024-11-08T00:43:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 77 connection: 172.17.0.2:42556 deadline: 1731026687443, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
2024-11-08T00:43:47,451 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T00:43:47,451 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it.
2024-11-08T00:43:47,456 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTable
2024-11-08T00:43:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,467 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-08T00:43:47,477 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,483 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/recovered.edits]
2024-11-08T00:43:47,497 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5/recovered.edits/4.seqid
2024-11-08T00:43:47,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTable/74ab50beefe6927e6e9d8ce049d56ee5
2024-11-08T00:43:47,498 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions
2024-11-08T00:43:47,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46797 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms
2024-11-08T00:43:47,516 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta
2024-11-08T00:43:47,550 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor.
2024-11-08T00:43:47,554 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,554 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states.
2024-11-08T00:43:47,555 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026627554"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:47,560 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-08T00:43:47,560 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 74ab50beefe6927e6e9d8ce049d56ee5, NAME => 'testDisableTable,,1731026625901.74ab50beefe6927e6e9d8ce049d56ee5.', STARTKEY => '', ENDKEY => ''}]
2024-11-08T00:43:47,560 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted.
2024-11-08T00:43:47,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731026627560"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:47,564 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META
2024-11-08T00:43:47,566 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:47,568 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 108 msec
2024-11-08T00:43:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-08T00:43:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable
2024-11-08T00:43:47,588 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed
2024-11-08T00:43:47,599 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=217 (was 207)
Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-434763642_22 at /127.0.0.1:41736 [Waiting for operation #6]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_OPEN_REGION-regionserver/c302dddf724e:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_CLOSE_REGION-regionserver/c302dddf724e:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 347), ProcessCount=11 (was 11), AvailableMemoryMB=2283 (was 2341)
2024-11-08T00:43:47,608 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=217, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=2283
2024-11-08T00:43:47,608 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:47,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:47,612 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T00:43:47,612 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:47,614 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T00:43:47,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741837_1013 (size=652)
2024-11-08T00:43:48,026 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0cb7bc9d2dc00a064d340dc63cc3566c, NAME => 'testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:48,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741838_1014 (size=64)
2024-11-08T00:43:48,435 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:48,436 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing 0cb7bc9d2dc00a064d340dc63cc3566c, disabling compactions & flushes
2024-11-08T00:43:48,436 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,436 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,436 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c. after waiting 0 ms
2024-11-08T00:43:48,436 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,436 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,436 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0cb7bc9d2dc00a064d340dc63cc3566c:
Waiting for close lock at 1731026628436Disabling compacts and flushes for region at 1731026628436Disabling writes for close at 1731026628436Writing region close event to WAL at 1731026628436Closed at 1731026628436
2024-11-08T00:43:48,438 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T00:43:48,438 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731026628438"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026628438"}]},"ts":"1731026628438"}
2024-11-08T00:43:48,441 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-08T00:43:48,443 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T00:43:48,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026628443"}]},"ts":"1731026628443"}
2024-11-08T00:43:48,446 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta
2024-11-08T00:43:48,446 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, ASSIGN}]
2024-11-08T00:43:48,449 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, ASSIGN
2024-11-08T00:43:48,450 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:48,601 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0cb7bc9d2dc00a064d340dc63cc3566c, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:48,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, ASSIGN because future has completed
2024-11-08T00:43:48,606 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:48,764 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,765 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 0cb7bc9d2dc00a064d340dc63cc3566c, NAME => 'testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.', STARTKEY => '', ENDKEY => ''}
2024-11-08T00:43:48,765 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,765 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:48,766 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,766 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,768 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 0cb7bc9d2dc00a064d340dc63cc3566c 
2024-11-08T00:43:48,770 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0cb7bc9d2dc00a064d340dc63cc3566c columnFamilyName f1
2024-11-08T00:43:48,770 DEBUG [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:48,770 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] regionserver.HStore(327): Store=0cb7bc9d2dc00a064d340dc63cc3566c/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:48,771 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 0cb7bc9d2dc00a064d340dc63cc3566c 
2024-11-08T00:43:48,773 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0cb7bc9d2dc00a064d340dc63cc3566c columnFamilyName f2
2024-11-08T00:43:48,773 DEBUG [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:48,774 INFO  [StoreOpener-0cb7bc9d2dc00a064d340dc63cc3566c-1 {}] regionserver.HStore(327): Store=0cb7bc9d2dc00a064d340dc63cc3566c/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:48,774 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,775 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,776 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,777 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,777 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,778 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:48,780 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,783 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:48,783 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 0cb7bc9d2dc00a064d340dc63cc3566c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67695332, jitterRate=0.008739054203033447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:48,784 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:48,785 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 0cb7bc9d2dc00a064d340dc63cc3566c:
Running coprocessor pre-open hook at 1731026628766Writing region info on filesystem at 1731026628766Initializing all the Stores at 1731026628767 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026628767Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026628767Cleaning up temporary data from old regions at 1731026628777 (+10 ms)Running coprocessor post-open hooks at 1731026628784 (+7 ms)Region opened successfully at 1731026628784
2024-11-08T00:43:48,786 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c., pid=15, masterSystemTime=1731026628759
2024-11-08T00:43:48,789 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,789 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:48,790 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0cb7bc9d2dc00a064d340dc63cc3566c, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:48,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:48,797 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14
2024-11-08T00:43:48,797 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161 in 189 msec
2024-11-08T00:43:48,801 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-08T00:43:48,801 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, ASSIGN in 351 msec
2024-11-08T00:43:48,802 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T00:43:48,803 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026628802"}]},"ts":"1731026628802"}
2024-11-08T00:43:48,805 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta
2024-11-08T00:43:48,807 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T00:43:48,809 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 1.1990 sec
2024-11-08T00:43:48,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:48,866 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026628866"}]},"ts":"1731026628866"}
2024-11-08T00:43:48,869 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta
2024-11-08T00:43:48,869 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING
2024-11-08T00:43:48,870 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}]
2024-11-08T00:43:48,873 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, UNASSIGN}]
2024-11-08T00:43:48,875 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, UNASSIGN
2024-11-08T00:43:48,877 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0cb7bc9d2dc00a064d340dc63cc3566c, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:48,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, UNASSIGN because future has completed
2024-11-08T00:43:48,880 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:48,880 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:49,033 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:49,034 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:49,034 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing 0cb7bc9d2dc00a064d340dc63cc3566c, disabling compactions & flushes
2024-11-08T00:43:49,034 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:49,034 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:49,034 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c. after waiting 0 ms
2024-11-08T00:43:49,034 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:49,040 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:49,041 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.
2024-11-08T00:43:49,041 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for 0cb7bc9d2dc00a064d340dc63cc3566c:
Waiting for close lock at 1731026629034Running coprocessor pre-close hooks at 1731026629034Disabling compacts and flushes for region at 1731026629034Disabling writes for close at 1731026629034Writing region close event to WAL at 1731026629034Running coprocessor post-close hooks at 1731026629041 (+7 ms)Closed at 1731026629041
2024-11-08T00:43:49,044 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed 0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:49,045 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0cb7bc9d2dc00a064d340dc63cc3566c, regionState=CLOSED
2024-11-08T00:43:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:49,053 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18
2024-11-08T00:43:49,053 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure 0cb7bc9d2dc00a064d340dc63cc3566c, server=c302dddf724e,46797,1731026622161 in 170 msec
2024-11-08T00:43:49,057 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17
2024-11-08T00:43:49,057 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=0cb7bc9d2dc00a064d340dc63cc3566c, UNASSIGN in 180 msec
2024-11-08T00:43:49,061 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16
2024-11-08T00:43:49,061 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 188 msec
2024-11-08T00:43:49,064 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026629063"}]},"ts":"1731026629063"}
2024-11-08T00:43:49,066 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta
2024-11-08T00:43:49,066 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED
2024-11-08T00:43:49,070 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 206 msec
2024-11-08T00:43:49,113 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-08T00:43:49,113 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:49,114 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:64) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:49,114 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:49,116 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes
2024-11-08T00:43:49,117 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTableMultipleTimes
2024-11-08T00:43:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.CallRunner(138): callId: 7 service: MasterService methodName: DisableTable size: 90 connection: 172.17.0.2:42556 deadline: 1731026689117, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-08T00:43:49,118 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T00:43:49,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it.
2024-11-08T00:43:49,119 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTableMultipleTimes
2024-11-08T00:43:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,122 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-08T00:43:49,123 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,127 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:49,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/recovered.edits]
2024-11-08T00:43:49,141 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c/recovered.edits/4.seqid
2024-11-08T00:43:49,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testDisableTableMultipleTimes/0cb7bc9d2dc00a064d340dc63cc3566c
2024-11-08T00:43:49,142 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions
2024-11-08T00:43:49,146 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,150 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta
2024-11-08T00:43:49,153 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor.
2024-11-08T00:43:49,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states.
2024-11-08T00:43:49,156 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026629156"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:49,159 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-08T00:43:49,159 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0cb7bc9d2dc00a064d340dc63cc3566c, NAME => 'testDisableTableMultipleTimes,,1731026627608.0cb7bc9d2dc00a064d340dc63cc3566c.', STARTKEY => '', ENDKEY => ''}]
2024-11-08T00:43:49,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted.
2024-11-08T00:43:49,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731026629159"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:49,162 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META
2024-11-08T00:43:49,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:49,166 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 45 msec
2024-11-08T00:43:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-08T00:43:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes
2024-11-08T00:43:49,230 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed
2024-11-08T00:43:49,237 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=222 (was 217)
Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-434763642_22 at /127.0.0.1:41736 [Waiting for operation #7]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=2280 (was 2283)
2024-11-08T00:43:49,246 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=2280
2024-11-08T00:43:49,246 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:49,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:49,250 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T00:43:49,250 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:49,251 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T00:43:49,259 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741839_1015 (size=653)
2024-11-08T00:43:49,662 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7c4b4699f494ba908283ad9d16a6c23, NAME => 'testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:49,663 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1c39cf7b170020f69bdadfd698d0830f, NAME => 'testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:49,663 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 24d35a011749709cd51a9cfe14e19e0b, NAME => 'testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:49,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741840_1016 (size=66)
2024-11-08T00:43:49,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741842_1018 (size=67)
2024-11-08T00:43:49,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741841_1017 (size=67)
2024-11-08T00:43:49,679 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:49,679 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 24d35a011749709cd51a9cfe14e19e0b, disabling compactions & flushes
2024-11-08T00:43:49,679 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:49,679 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:49,679 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b. after waiting 0 ms
2024-11-08T00:43:49,679 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:49,679 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:49,680 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 24d35a011749709cd51a9cfe14e19e0b:
Waiting for close lock at 1731026629679Disabling compacts and flushes for region at 1731026629679Disabling writes for close at 1731026629679Writing region close event to WAL at 1731026629679Closed at 1731026629679
2024-11-08T00:43:49,680 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4c8b57d039be6576957a57f56d8ba63a, NAME => 'testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269
2024-11-08T00:43:49,686 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741843_1019 (size=66)
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing 1c39cf7b170020f69bdadfd698d0830f, disabling compactions & flushes
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing e7c4b4699f494ba908283ad9d16a6c23, disabling compactions & flushes
2024-11-08T00:43:50,079 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,079 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f. after waiting 0 ms
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23. after waiting 0 ms
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,079 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,079 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1c39cf7b170020f69bdadfd698d0830f:
Waiting for close lock at 1731026630079Disabling compacts and flushes for region at 1731026630079Disabling writes for close at 1731026630079Writing region close event to WAL at 1731026630079Closed at 1731026630079
2024-11-08T00:43:50,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7c4b4699f494ba908283ad9d16a6c23:
Waiting for close lock at 1731026630079Disabling compacts and flushes for region at 1731026630079Disabling writes for close at 1731026630079Writing region close event to WAL at 1731026630079Closed at 1731026630079
2024-11-08T00:43:50,087 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,087 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 4c8b57d039be6576957a57f56d8ba63a, disabling compactions & flushes
2024-11-08T00:43:50,087 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,088 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,088 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a. after waiting 0 ms
2024-11-08T00:43:50,088 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,088 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,088 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4c8b57d039be6576957a57f56d8ba63a:
Waiting for close lock at 1731026630087Disabling compacts and flushes for region at 1731026630087Disabling writes for close at 1731026630088 (+1 ms)Writing region close event to WAL at 1731026630088Closed at 1731026630088
2024-11-08T00:43:50,090 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T00:43:50,090 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731026630090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026630090"}]},"ts":"1731026630090"}
2024-11-08T00:43:50,090 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731026630090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026630090"}]},"ts":"1731026630090"}
2024-11-08T00:43:50,090 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731026630090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026630090"}]},"ts":"1731026630090"}
2024-11-08T00:43:50,090 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731026630090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731026630090"}]},"ts":"1731026630090"}
2024-11-08T00:43:50,095 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta.
2024-11-08T00:43:50,096 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T00:43:50,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026630097"}]},"ts":"1731026630097"}
2024-11-08T00:43:50,099 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta
2024-11-08T00:43:50,099 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, ASSIGN}]
2024-11-08T00:43:50,102 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, ASSIGN
2024-11-08T00:43:50,103 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:50,104 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, ASSIGN
2024-11-08T00:43:50,105 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:50,106 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, ASSIGN
2024-11-08T00:43:50,106 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:50,107 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, ASSIGN
2024-11-08T00:43:50,108 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, ASSIGN; state=OFFLINE, location=c302dddf724e,46797,1731026622161; forceNewPlan=false, retain=false
2024-11-08T00:43:50,254 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,255 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,255 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,256 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPENING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, ASSIGN because future has completed
2024-11-08T00:43:50,258 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:50,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, ASSIGN because future has completed
2024-11-08T00:43:50,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, ASSIGN because future has completed
2024-11-08T00:43:50,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, ASSIGN because future has completed
2024-11-08T00:43:50,262 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:50,266 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:50,269 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:50,416 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,416 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => e7c4b4699f494ba908283ad9d16a6c23, NAME => 'testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.', STARTKEY => '', ENDKEY => 'a'}
2024-11-08T00:43:50,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,417 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,419 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region e7c4b4699f494ba908283ad9d16a6c23 
2024-11-08T00:43:50,422 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7c4b4699f494ba908283ad9d16a6c23 columnFamilyName f1
2024-11-08T00:43:50,422 DEBUG [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,422 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] regionserver.HStore(327): Store=e7c4b4699f494ba908283ad9d16a6c23/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,423 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region e7c4b4699f494ba908283ad9d16a6c23 
2024-11-08T00:43:50,424 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7c4b4699f494ba908283ad9d16a6c23 columnFamilyName f2
2024-11-08T00:43:50,424 DEBUG [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,425 INFO  [StoreOpener-e7c4b4699f494ba908283ad9d16a6c23-1 {}] regionserver.HStore(327): Store=e7c4b4699f494ba908283ad9d16a6c23/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,425 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,427 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,427 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,429 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:50,431 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,435 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:50,435 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened e7c4b4699f494ba908283ad9d16a6c23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67032622, jitterRate=-0.001136094331741333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:50,436 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:50,437 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for e7c4b4699f494ba908283ad9d16a6c23:
Running coprocessor pre-open hook at 1731026630417Writing region info on filesystem at 1731026630417Initializing all the Stores at 1731026630419 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630419Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630419Cleaning up temporary data from old regions at 1731026630429 (+10 ms)Running coprocessor post-open hooks at 1731026630436 (+7 ms)Region opened successfully at 1731026630437 (+1 ms)
2024-11-08T00:43:50,439 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23., pid=28, masterSystemTime=1731026630410
2024-11-08T00:43:50,442 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,442 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:50,442 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,443 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c39cf7b170020f69bdadfd698d0830f, NAME => 'testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.', STARTKEY => 'b', ENDKEY => 'c'}
2024-11-08T00:43:50,443 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,443 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,443 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,443 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,444 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:50,446 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 1c39cf7b170020f69bdadfd698d0830f 
2024-11-08T00:43:50,448 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c39cf7b170020f69bdadfd698d0830f columnFamilyName f1
2024-11-08T00:43:50,448 DEBUG [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,449 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] regionserver.HStore(327): Store=1c39cf7b170020f69bdadfd698d0830f/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,449 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 1c39cf7b170020f69bdadfd698d0830f 
2024-11-08T00:43:50,450 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=23
2024-11-08T00:43:50,451 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 in 185 msec
2024-11-08T00:43:50,452 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c39cf7b170020f69bdadfd698d0830f columnFamilyName f2
2024-11-08T00:43:50,452 DEBUG [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,453 INFO  [StoreOpener-1c39cf7b170020f69bdadfd698d0830f-1 {}] regionserver.HStore(327): Store=1c39cf7b170020f69bdadfd698d0830f/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,453 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, ASSIGN in 352 msec
2024-11-08T00:43:50,453 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,454 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,454 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,455 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,455 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,456 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:50,458 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,461 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:50,461 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 1c39cf7b170020f69bdadfd698d0830f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71153378, jitterRate=0.06026795506477356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:50,462 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:50,462 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1c39cf7b170020f69bdadfd698d0830f:
Running coprocessor pre-open hook at 1731026630444Writing region info on filesystem at 1731026630444Initializing all the Stores at 1731026630445 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630445Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630446 (+1 ms)Cleaning up temporary data from old regions at 1731026630455 (+9 ms)Running coprocessor post-open hooks at 1731026630462 (+7 ms)Region opened successfully at 1731026630462
2024-11-08T00:43:50,463 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f., pid=30, masterSystemTime=1731026630410
2024-11-08T00:43:50,465 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,466 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:50,466 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,466 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c8b57d039be6576957a57f56d8ba63a, NAME => 'testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.', STARTKEY => 'c', ENDKEY => ''}
2024-11-08T00:43:50,466 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,466 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,467 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,467 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,467 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,469 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 4c8b57d039be6576957a57f56d8ba63a 
2024-11-08T00:43:50,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:50,471 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c8b57d039be6576957a57f56d8ba63a columnFamilyName f1
2024-11-08T00:43:50,471 DEBUG [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,471 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] regionserver.HStore(327): Store=4c8b57d039be6576957a57f56d8ba63a/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,472 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 4c8b57d039be6576957a57f56d8ba63a 
2024-11-08T00:43:50,474 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c8b57d039be6576957a57f56d8ba63a columnFamilyName f2
2024-11-08T00:43:50,474 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=25
2024-11-08T00:43:50,474 DEBUG [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,474 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 in 201 msec
2024-11-08T00:43:50,474 INFO  [StoreOpener-4c8b57d039be6576957a57f56d8ba63a-1 {}] regionserver.HStore(327): Store=4c8b57d039be6576957a57f56d8ba63a/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,475 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,476 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, ASSIGN in 375 msec
2024-11-08T00:43:50,476 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,476 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,477 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,477 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,478 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:50,480 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,482 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:50,483 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened 4c8b57d039be6576957a57f56d8ba63a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64021355, jitterRate=-0.04600746929645538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:50,483 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:50,483 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 4c8b57d039be6576957a57f56d8ba63a:
Running coprocessor pre-open hook at 1731026630467Writing region info on filesystem at 1731026630467Initializing all the Stores at 1731026630468 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630468Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630469 (+1 ms)Cleaning up temporary data from old regions at 1731026630477 (+8 ms)Running coprocessor post-open hooks at 1731026630483 (+6 ms)Region opened successfully at 1731026630483
2024-11-08T00:43:50,484 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a., pid=27, masterSystemTime=1731026630410
2024-11-08T00:43:50,486 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,486 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:50,487 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:50,487 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => 24d35a011749709cd51a9cfe14e19e0b, NAME => 'testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.', STARTKEY => 'a', ENDKEY => 'b'}
2024-11-08T00:43:50,487 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,488 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,488 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T00:43:50,488 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,488 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,490 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 24d35a011749709cd51a9cfe14e19e0b 
2024-11-08T00:43:50,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:50,492 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24d35a011749709cd51a9cfe14e19e0b columnFamilyName f1
2024-11-08T00:43:50,492 DEBUG [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,493 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] regionserver.HStore(327): Store=24d35a011749709cd51a9cfe14e19e0b/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,493 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 24d35a011749709cd51a9cfe14e19e0b 
2024-11-08T00:43:50,494 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26
2024-11-08T00:43:50,494 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 in 234 msec
2024-11-08T00:43:50,495 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24d35a011749709cd51a9cfe14e19e0b columnFamilyName f2
2024-11-08T00:43:50,495 DEBUG [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T00:43:50,496 INFO  [StoreOpener-24d35a011749709cd51a9cfe14e19e0b-1 {}] regionserver.HStore(327): Store=24d35a011749709cd51a9cfe14e19e0b/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T00:43:50,496 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, ASSIGN in 395 msec
2024-11-08T00:43:50,497 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,498 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,499 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,499 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,499 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T00:43:50,501 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,504 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T00:43:50,504 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened 24d35a011749709cd51a9cfe14e19e0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60312485, jitterRate=-0.10127393901348114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T00:43:50,505 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:50,505 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for 24d35a011749709cd51a9cfe14e19e0b:
Running coprocessor pre-open hook at 1731026630488Writing region info on filesystem at 1731026630488Initializing all the Stores at 1731026630490 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630490Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731026630490Cleaning up temporary data from old regions at 1731026630499 (+9 ms)Running coprocessor post-open hooks at 1731026630505 (+6 ms)Region opened successfully at 1731026630505
2024-11-08T00:43:50,506 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b., pid=29, masterSystemTime=1731026630410
2024-11-08T00:43:50,508 DEBUG [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:50,508 INFO  [RS_OPEN_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:50,509 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, openSeqNum=2, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:50,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:50,516 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=24
2024-11-08T00:43:50,516 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 in 247 msec
2024-11-08T00:43:50,519 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22
2024-11-08T00:43:50,519 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, ASSIGN in 417 msec
2024-11-08T00:43:50,520 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T00:43:50,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026630520"}]},"ts":"1731026630520"}
2024-11-08T00:43:50,523 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta
2024-11-08T00:43:50,524 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T00:43:50,526 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.2780 sec
2024-11-08T00:43:50,749 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true
2024-11-08T00:43:50,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:50,752 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:50,752 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:50,752 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:50,753 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:51,001 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,002 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:51,002 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,002 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:51,002 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:51,003 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-1,5,PEWorkerGroup]
    Thread[HFileArchiver-2,5,PEWorkerGroup]
2024-11-08T00:43:51,003 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:51,003 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:51,003 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:51,004 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:51,005 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:51,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[]
2024-11-08T00:43:51,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:51,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:51,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:51,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:51,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,011 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:51,012 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec
2024-11-08T00:43:51,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,014 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:51,014 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,018 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026661018
2024-11-08T00:43:51,019 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:51,019 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,026 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:51,026 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026691026
2024-11-08T00:43:51,026 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026751026
2024-11-08T00:43:51,026 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec
2024-11-08T00:43:51,027 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:51,028 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:51,028 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,028 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,028 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:51,085 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-08T00:43:51,133 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-08T00:43:51,135 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution'
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:51,277 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:51,278 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:51,279 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:51,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0]
2024-11-08T00:43:51,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:51,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:51,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:51,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:51,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:51,286 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec
2024-11-08T00:43:51,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:51,288 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:51,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,291 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026661291
2024-11-08T00:43:51,291 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:51,291 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,297 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:51,297 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026691297
2024-11-08T00:43:51,297 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026751297
2024-11-08T00:43:51,297 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T00:43:51,297 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:51,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026631299"}]},"ts":"1731026631299"}
2024-11-08T00:43:51,301 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-08T00:43:51,301 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-08T00:43:51,301 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:51,301 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,301 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,301 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:51,547 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,547 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:51,547 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:51,548 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:51,549 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:51,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1]
2024-11-08T00:43:51,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:51,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:51,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:51,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:51,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:51,555 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:51,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,556 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:51,556 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:51,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,559 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026661559
2024-11-08T00:43:51,559 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:51,559 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,564 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:51,564 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026691564
2024-11-08T00:43:51,564 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026751564
2024-11-08T00:43:51,564 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:51,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026631564"}]},"ts":"1731026631564"}
2024-11-08T00:43:51,567 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-08T00:43:51,567 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-08T00:43:51,567 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:51,568 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-08T00:43:51,568 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:51,568 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,568 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,569 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:51,814 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,814 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:51,814 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,814 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:51,815 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:51,815 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:51,815 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:51,815 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:51,815 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:51,816 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:51,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2]
2024-11-08T00:43:51,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:51,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:51,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:51,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:51,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:51,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:51,821 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:51,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,822 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:51,822 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:51,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:51,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:51,826 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026661826
2024-11-08T00:43:51,826 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:51,826 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:51,831 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:51,832 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026691831
2024-11-08T00:43:51,832 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026751832
2024-11-08T00:43:51,832 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T00:43:51,832 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-08T00:43:51,832 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:51,836 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN}]
2024-11-08T00:43:51,836 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:51,836 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:51,836 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:51,836 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:51,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-08T00:43:51,878 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-08T00:43:51,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-08T00:43:51,880 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-08T00:43:51,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-08T00:43:51,880 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-08T00:43:51,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution
2024-11-08T00:43:51,881 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:52,082 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:52,083 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:52,084 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:52,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:52,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[]
2024-11-08T00:43:52,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:52,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:52,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:52,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:52,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,089 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:52,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,090 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:52,090 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:52,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,093 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026662093
2024-11-08T00:43:52,093 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:52,093 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,098 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:52,098 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026692098
2024-11-08T00:43:52,098 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026752098
2024-11-08T00:43:52,098 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T00:43:52,099 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN}]
2024-11-08T00:43:52,099 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:52,102 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,103 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,104 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,105 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,106 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:52,107 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,107 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:52,107 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:52,108 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:52,108 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,108 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,108 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:52,108 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f61c4bd7d78@66f25f7f rejected from java.util.concurrent.ThreadPoolExecutor@17b07191[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:52,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN because future has completed
2024-11-08T00:43:52,110 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f61c4bd7d78@117321d4 rejected from java.util.concurrent.ThreadPoolExecutor@17b07191[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:52,120 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,120 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,223 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,223 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,326 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,326 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,348 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:52,349 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:52,351 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:52,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:52,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:52,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:52,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:52,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:52,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:52,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,359 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,359 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,359 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,359 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,359 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,359 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec
2024-11-08T00:43:52,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:52,360 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:52,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,363 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23 to restore RIT
2024-11-08T00:43:52,363 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:52,363 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:52,363 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:52,364 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026662364
2024-11-08T00:43:52,364 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:52,364 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,364 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,364 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,364 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,368 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:52,368 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026692368
2024-11-08T00:43:52,368 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026752368
2024-11-08T00:43:52,368 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:52,369 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,370 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,371 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,371 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,372 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN because future has completed
2024-11-08T00:43:52,373 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:52,373 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:52,373 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:52,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN because future has completed
2024-11-08T00:43:52,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:52,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:52,377 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:52,378 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:52,378 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:52,378 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,378 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,378 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:52,428 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,429 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,527 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:52,527 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:52,527 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing e7c4b4699f494ba908283ad9d16a6c23, disabling compactions & flushes
2024-11-08T00:43:52,527 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:52,527 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:52,527 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23. after waiting 0 ms
2024-11-08T00:43:52,527 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:52,531 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,531 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,533 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:52,535 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.
2024-11-08T00:43:52,535 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for e7c4b4699f494ba908283ad9d16a6c23:
Waiting for close lock at 1731026632527Running coprocessor pre-close hooks at 1731026632527Disabling compacts and flushes for region at 1731026632527Disabling writes for close at 1731026632527Writing region close event to WAL at 1731026632528 (+1 ms)Running coprocessor post-close hooks at 1731026632534 (+6 ms)Closed at 1731026632534
2024-11-08T00:43:52,537 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:52,619 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:52,620 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:52,621 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:52,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:52,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:52,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:52,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,629 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,629 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,629 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,629 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,630 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,630 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,630 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,630 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec
2024-11-08T00:43:52,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,631 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:52,631 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:52,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,634 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:52,636 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23 to restore RIT
2024-11-08T00:43:52,636 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:52,636 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:52,636 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:52,643 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026662643
2024-11-08T00:43:52,644 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:52,644 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,644 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,644 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,644 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,650 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:52,650 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026692650
2024-11-08T00:43:52,650 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026752650
2024-11-08T00:43:52,650 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-08T00:43:52,652 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED
2024-11-08T00:43:52,654 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,655 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:52,655 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,658 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:52,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN because future has completed
2024-11-08T00:43:52,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:52,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:52,664 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33
2024-11-08T00:43:52,664 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 in 285 msec
2024-11-08T00:43:52,664 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:52,664 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,664 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,664 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:52,739 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,739 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,739 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,842 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,842 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,842 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,900 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:52,901 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:52,902 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:52,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:52,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:52,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:52,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:52,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:52,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:52,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,907 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:52,907 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,908 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,908 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:52,908 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,908 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:52,909 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-08T00:43:52,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,909 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:52,910 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:52,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:52,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:52,914 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23 to restore RIT
2024-11-08T00:43:52,914 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:52,914 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:52,914 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:52,914 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026662914
2024-11-08T00:43:52,914 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:52,914 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,914 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,914 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:52,914 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:52,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:52,918 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:52,918 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026692918
2024-11-08T00:43:52,918 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026752918
2024-11-08T00:43:52,919 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:52,919 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:52,920 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:52,922 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN in 822 msec
2024-11-08T00:43:52,922 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,923 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,923 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:52,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN because future has completed
2024-11-08T00:43:52,926 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:52,926 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:52,926 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:52,926 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:52,926 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:52,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:52,926 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:52,926 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f61c4bd7d78@6cfbe065 rejected from java.util.concurrent.ThreadPoolExecutor@5628abd3[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:52,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:52,927 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f61c4bd7d78@4010af05 rejected from java.util.concurrent.ThreadPoolExecutor@5628abd3[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:52,945 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:52,945 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,048 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,048 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,151 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,151 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:53,169 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:53,170 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:53,171 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:53,173 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:53,173 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:53,173 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:53,173 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:53,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:53,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:53,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:53,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:53,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,178 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,178 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,178 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,178 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec
2024-11-08T00:43:53,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,179 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:53,180 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:53,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,183 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:53,183 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:53,183 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:53,183 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026663183
2024-11-08T00:43:53,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:53,183 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,183 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,184 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,188 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:53,188 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026693188
2024-11-08T00:43:53,188 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026753188
2024-11-08T00:43:53,188 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:53,188 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,189 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,189 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN because future has completed
2024-11-08T00:43:53,191 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:53,191 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:53,191 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:53,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:53,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:53,195 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:53,195 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:53,195 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:53,195 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,195 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,195 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:53,254 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,345 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:53,345 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:53,345 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 24d35a011749709cd51a9cfe14e19e0b, disabling compactions & flushes
2024-11-08T00:43:53,345 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:53,345 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:53,345 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b. after waiting 0 ms
2024-11-08T00:43:53,345 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:53,352 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:53,353 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.
2024-11-08T00:43:53,353 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 24d35a011749709cd51a9cfe14e19e0b:
Waiting for close lock at 1731026633345Running coprocessor pre-close hooks at 1731026633345Disabling compacts and flushes for region at 1731026633345Disabling writes for close at 1731026633345Writing region close event to WAL at 1731026633346 (+1 ms)Running coprocessor post-close hooks at 1731026633353 (+7 ms)Closed at 1731026633353
2024-11-08T00:43:53,357 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:53,358 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,438 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,438 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:53,438 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,438 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:53,438 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:53,439 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:53,439 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:53,439 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:53,439 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:53,440 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:53,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:53,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:53,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:53,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:53,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,448 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,448 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,448 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,448 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,448 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec
2024-11-08T00:43:53,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,450 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:53,450 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:53,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,452 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,452 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,453 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:53,453 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:53,453 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:53,453 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026663453
2024-11-08T00:43:53,453 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:53,453 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,453 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,454 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,457 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,458 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:53,458 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026693458
2024-11-08T00:43:53,458 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026753458
2024-11-08T00:43:53,458 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:53,460 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED
2024-11-08T00:43:53,461 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,462 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:53,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:53,464 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:53,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:53,466 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=34
2024-11-08T00:43:53,466 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 in 273 msec
2024-11-08T00:43:53,467 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:53,467 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,467 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,467 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:53,563 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,563 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,666 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,666 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:53,708 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:53,709 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:53,710 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,715 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,715 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,715 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:53,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,716 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:53,717 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:53,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,719 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b to restore RIT
2024-11-08T00:43:53,719 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:53,719 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:53,720 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026663720
2024-11-08T00:43:53,720 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:53,720 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,720 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,720 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,724 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:53,724 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026693724
2024-11-08T00:43:53,724 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026753724
2024-11-08T00:43:53,724 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:53,724 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,726 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:53,728 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN in 1.6280 sec
2024-11-08T00:43:53,728 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,729 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:53,731 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:53,732 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:53,732 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:53,732 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,732 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,732 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:53,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:53,732 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f61c4bd7d78@3f5fbc78 rejected from java.util.concurrent.ThreadPoolExecutor@30659402[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T00:43:53,769 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,871 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,973 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:53,974 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,974 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:53,974 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,974 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:53,974 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:53,975 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:53,975 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:53,975 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:53,975 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:53,976 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:53,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:53,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,982 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,982 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:53,982 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,982 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,982 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:53,982 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:53,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,982 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:53,983 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:53,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:53,985 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,985 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:53,985 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:53,986 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:53,986 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026663986
2024-11-08T00:43:53,986 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:53,986 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:53,986 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:53,990 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:53,990 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026693990
2024-11-08T00:43:53,990 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026753990
2024-11-08T00:43:53,990 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:53,990 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,991 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:53,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN because future has completed
2024-11-08T00:43:53,993 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:53,993 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:53,993 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:53,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:53,996 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:53,997 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:53,997 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:53,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:53,997 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:53,997 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:54,146 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:54,146 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:54,147 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing 1c39cf7b170020f69bdadfd698d0830f, disabling compactions & flushes
2024-11-08T00:43:54,147 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:54,147 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:54,147 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f. after waiting 0 ms
2024-11-08T00:43:54,147 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:54,153 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:54,154 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.
2024-11-08T00:43:54,154 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for 1c39cf7b170020f69bdadfd698d0830f:
Waiting for close lock at 1731026634147Running coprocessor pre-close hooks at 1731026634147Disabling compacts and flushes for region at 1731026634147Disabling writes for close at 1731026634147Writing region close event to WAL at 1731026634147Running coprocessor post-close hooks at 1731026634154 (+7 ms)Closed at 1731026634154
2024-11-08T00:43:54,157 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed 1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:54,240 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,240 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:54,240 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,240 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:54,240 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:54,241 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:54,241 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:54,241 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:54,241 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:54,242 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:54,243 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:54,244 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:54,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:54,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:54,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:54,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:54,247 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:54,247 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,247 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:54,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,248 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:54,248 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:54,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,251 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:54,251 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:54,252 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026664252
2024-11-08T00:43:54,252 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:54,252 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,252 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,256 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:54,256 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026694256
2024-11-08T00:43:54,256 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026754256
2024-11-08T00:43:54,256 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:54,257 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED
2024-11-08T00:43:54,259 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:54,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:54,262 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:54,265 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=35
2024-11-08T00:43:54,265 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 in 269 msec
2024-11-08T00:43:54,265 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:54,265 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,265 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,265 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:54,281 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:54,383 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:54,486 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:54,506 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,506 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:54,506 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,506 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:54,506 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:54,507 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:54,507 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:54,507 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:54,507 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:54,508 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:54,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:54,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:54,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:54,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:54,515 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,515 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-08T00:43:54,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,516 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:54,516 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:54,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,520 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,520 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,521 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f to restore RIT
2024-11-08T00:43:54,521 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:54,521 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026664521
2024-11-08T00:43:54,521 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:54,522 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,522 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,526 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:54,526 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026694526
2024-11-08T00:43:54,526 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026754526
2024-11-08T00:43:54,526 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:54,526 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,527 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:54,529 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN in 2.4290 sec
2024-11-08T00:43:54,529 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:54,533 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:54,533 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:54,533 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:54,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,533 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,533 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:54,776 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:54,777 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:54,777 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:54,777 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:54,778 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:54,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T00:43:54,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:54,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:54,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:54,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:54,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:54,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:54,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:54,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:54,783 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:54,783 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:54,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,784 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:54,784 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:54,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:54,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:54,787 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:54,787 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026664787
2024-11-08T00:43:54,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:54,787 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:54,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:54,791 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:54,791 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026694791
2024-11-08T00:43:54,791 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026754791
2024-11-08T00:43:54,791 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:54,791 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSING, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,793 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44077 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c302dddf724e,46797,1731026622161, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T00:43:54,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN because future has completed
2024-11-08T00:43:54,794 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T00:43:54,794 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161}]
2024-11-08T00:43:54,794 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:54,948 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:54,948 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T00:43:54,948 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 4c8b57d039be6576957a57f56d8ba63a, disabling compactions & flushes
2024-11-08T00:43:54,948 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:54,948 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:54,948 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a. after waiting 0 ms
2024-11-08T00:43:54,948 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:54,954 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T00:43:54,955 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.
2024-11-08T00:43:54,955 DEBUG [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 4c8b57d039be6576957a57f56d8ba63a:
Waiting for close lock at 1731026634948Running coprocessor pre-close hooks at 1731026634948Disabling compacts and flushes for region at 1731026634948Disabling writes for close at 1731026634948Writing region close event to WAL at 1731026634949 (+1 ms)Running coprocessor post-close hooks at 1731026634955 (+6 ms)Closed at 1731026634955
2024-11-08T00:43:54,958 INFO  [RS_CLOSE_REGION-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:54,958 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED
2024-11-08T00:43:54,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:54,962 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:54,962 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:54,962 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:54,962 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:55,041 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,041 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:55,041 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,041 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:55,041 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:55,042 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:55,042 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:55,042 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:55,042 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:55,043 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-08T00:43:55,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18]
2024-11-08T00:43:55,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:55,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:55,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:55,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:55,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:55,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:55,048 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:55,048 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T00:43:55,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,049 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:55,049 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:55,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,052 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,052 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,052 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:55,053 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026665052
2024-11-08T00:43:55,053 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:55,053 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,057 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:55,057 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026695057
2024-11-08T00:43:55,057 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026755057
2024-11-08T00:43:55,057 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T00:43:55,058 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, regionLocation=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 because future has completed
2024-11-08T00:43:55,062 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:55,064 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=36
2024-11-08T00:43:55,064 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 in 268 msec
2024-11-08T00:43:55,064 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:55,064 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,064 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,064 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:55,308 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:55,309 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:55,310 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T00:43:55,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,313 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,314 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:55,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,315 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:55,315 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:55,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,318 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a to restore RIT
2024-11-08T00:43:55,319 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026665318
2024-11-08T00:43:55,319 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:55,319 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,322 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:55,322 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026695322
2024-11-08T00:43:55,322 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026755322
2024-11-08T00:43:55,322 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:55,323 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,324 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:55,326 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=32
2024-11-08T00:43:55,326 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN in 3.2260 sec
2024-11-08T00:43:55,327 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:55,327 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,327 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,327 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:55,573 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:55,574 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:55,575 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T00:43:55,576 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:55,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,579 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:55,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,580 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:55,580 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:55,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,584 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026665584
2024-11-08T00:43:55,584 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:55,584 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,588 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:55,588 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026695588
2024-11-08T00:43:55,588 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026755588
2024-11-08T00:43:55,588 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:55,588 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:55,591 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31
2024-11-08T00:43:55,591 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 3.7560 sec
2024-11-08T00:43:55,592 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:55,592 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,592 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,592 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:55,838 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,838 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:55,838 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,838 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:55,838 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:55,839 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:55,839 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:55,839 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:55,839 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:55,840 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T00:43:55,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T00:43:55,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:55,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:55,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:55,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:55,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:55,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:55,845 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:55,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,845 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:55,846 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:55,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:55,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:55,849 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026665849
2024-11-08T00:43:55,849 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:55,849 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:55,852 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:55,852 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026695852
2024-11-08T00:43:55,852 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026755852
2024-11-08T00:43:55,853 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:55,853 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:55,854 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026635854"}]},"ts":"1731026635854"}
2024-11-08T00:43:55,856 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-08T00:43:55,857 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-08T00:43:55,857 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:55,857 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:55,857 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:55,857 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:55,990 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:56,103 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:56,104 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:56,105 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T00:43:56,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,109 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,110 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:56,111 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:56,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:56,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:56,113 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:56,113 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:56,114 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026666114
2024-11-08T00:43:56,114 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:56,114 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,118 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:56,118 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026696118
2024-11-08T00:43:56,118 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026756118
2024-11-08T00:43:56,118 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:56,118 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731026636118"}]},"ts":"1731026636118"}
2024-11-08T00:43:56,124 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-08T00:43:56,124 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-08T00:43:56,124 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:56,125 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T00:43:56,125 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,125 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:56,125 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:56,368 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,368 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T00:43:56,368 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:56,368 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:56,368 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:56,369 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T00:43:56,369 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T00:43:56,369 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T00:43:56,369 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T00:43:56,370 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T00:43:56,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23]
2024-11-08T00:43:56,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T00:43:56,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T00:43:56,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T00:43:56,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T00:43:56,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T00:43:56,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T00:43:56,374 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=e7c4b4699f494ba908283ad9d16a6c23, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=24d35a011749709cd51a9cfe14e19e0b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1c39cf7b170020f69bdadfd698d0830f, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4c8b57d039be6576957a57f56d8ba63a, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e7c4b4699f494ba908283ad9d16a6c23, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 24d35a011749709cd51a9cfe14e19e0b, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c39cf7b170020f69bdadfd698d0830f, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c8b57d039be6576957a57f56d8ba63a, server=c302dddf724e,46797,1731026622161 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T00:43:56,375 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,375 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c302dddf724e,46797,1731026622161, table=hbase:meta, region=1588230740
2024-11-08T00:43:56,376 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c302dddf724e,46797,1731026622161, state=OPEN
2024-11-08T00:43:56,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:56,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T00:43:56,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:56,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T00:43:56,379 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731026666378
2024-11-08T00:43:56,379 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T00:43:56,379 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7c4b4699f494ba908283ad9d16a6c23, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24d35a011749709cd51a9cfe14e19e0b, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c39cf7b170020f69bdadfd698d0830f, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8b57d039be6576957a57f56d8ba63a, regionState=CLOSED, lastHost=c302dddf724e,46797,1731026622161, regionLocation=c302dddf724e,46797,1731026622161, openSeqNum=2
2024-11-08T00:43:56,382 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T00:43:56,382 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731026696382
2024-11-08T00:43:56,382 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731026756382
2024-11-08T00:43:56,382 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T00:43:56,382 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T00:43:56,385 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 5.6330 sec
2024-11-08T00:43:56,633 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T00:43:56,635 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,635 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testRecoveryAndDoubleExecution
2024-11-08T00:43:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T00:43:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] ipc.CallRunner(138): callId: 12 service: MasterService methodName: DisableTable size: 91 connection: 172.17.0.2:42556 deadline: 1731026696635, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
2024-11-08T00:43:56,637 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T00:43:56,637 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it.
2024-11-08T00:43:56,638 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testRecoveryAndDoubleExecution
2024-11-08T00:43:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,640 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-08T00:43:56,642 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,650 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:56,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:56,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:56,650 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:56,654 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/recovered.edits]
2024-11-08T00:43:56,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/recovered.edits]
2024-11-08T00:43:56,654 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/recovered.edits]
2024-11-08T00:43:56,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/f1, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/f2, FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/recovered.edits]
2024-11-08T00:43:56,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23/recovered.edits/4.seqid
2024-11-08T00:43:56,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a/recovered.edits/4.seqid
2024-11-08T00:43:56,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f/recovered.edits/4.seqid
2024-11-08T00:43:56,667 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/recovered.edits/4.seqid to hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/archive/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b/recovered.edits/4.seqid
2024-11-08T00:43:56,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/e7c4b4699f494ba908283ad9d16a6c23
2024-11-08T00:43:56,667 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/4c8b57d039be6576957a57f56d8ba63a
2024-11-08T00:43:56,667 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/1c39cf7b170020f69bdadfd698d0830f
2024-11-08T00:43:56,667 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/default/testRecoveryAndDoubleExecution/24d35a011749709cd51a9cfe14e19e0b
2024-11-08T00:43:56,668 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions
2024-11-08T00:43:56,671 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,676 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta
2024-11-08T00:43:56,679 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor.
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states.
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026636681"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026636681"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026636681"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:56,681 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731026636681"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:56,684 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META
2024-11-08T00:43:56,684 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7c4b4699f494ba908283ad9d16a6c23, NAME => 'testRecoveryAndDoubleExecution,,1731026629246.e7c4b4699f494ba908283ad9d16a6c23.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => 24d35a011749709cd51a9cfe14e19e0b, NAME => 'testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => 1c39cf7b170020f69bdadfd698d0830f, NAME => 'testRecoveryAndDoubleExecution,b,1731026629246.1c39cf7b170020f69bdadfd698d0830f.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => 4c8b57d039be6576957a57f56d8ba63a, NAME => 'testRecoveryAndDoubleExecution,c,1731026629246.4c8b57d039be6576957a57f56d8ba63a.', STARTKEY => 'c', ENDKEY => ''}]
2024-11-08T00:43:56,684 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted.
2024-11-08T00:43:56,684 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731026636684"}]},"ts":"9223372036854775807"}
2024-11-08T00:43:56,686 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META
2024-11-08T00:43:56,687 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T00:43:56,689 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 49 msec
2024-11-08T00:43:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-08T00:43:56,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution
2024-11-08T00:43:56,747 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed
2024-11-08T00:43:56,756 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=219 (was 222), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=2244 (was 2280)
2024-11-08T00:43:56,756 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-08T00:43:56,757 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T00:43:56,757 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T00:43:56,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:56,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:56,764 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-08T00:43:56,764 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-08T00:43:56,764 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1510831694, stopped=false
2024-11-08T00:43:56,764 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c302dddf724e,44077,1731026621470
2024-11-08T00:43:56,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T00:43:56,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T00:43:56,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:56,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:56,767 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:56,767 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T00:43:56,767 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T00:43:56,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:56,768 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c302dddf724e,46797,1731026622161' *****
2024-11-08T00:43:56,768 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T00:43:56,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T00:43:56,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T00:43:56,768 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-08T00:43:56,769 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(959): stopping server c302dddf724e,46797,1731026622161
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c302dddf724e:46797.
2024-11-08T00:43:56,769 DEBUG [RS:0;c302dddf724e:46797 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T00:43:56,769 DEBUG [RS:0;c302dddf724e:46797 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-08T00:43:56,769 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-08T00:43:56,770 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-08T00:43:56,770 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-08T00:43:56,770 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T00:43:56,770 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:56,770 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T00:43:56,770 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T00:43:56,771 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T00:43:56,771 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T00:43:56,774 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=25.47 KB heapSize=41.64 KB
2024-11-08T00:43:56,820 INFO  [regionserver/c302dddf724e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T00:43:56,840 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/info/3c6c493024c441ecac2e62c2873c86bf is 108, key is testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b./info:/1731026636681/DeleteFamily/seqid=0
2024-11-08T00:43:56,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741844_1020 (size=6669)
2024-11-08T00:43:56,971 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:57,016 INFO  [regionserver/c302dddf724e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-08T00:43:57,016 INFO  [regionserver/c302dddf724e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-08T00:43:57,171 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:57,252 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.63 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/info/3c6c493024c441ecac2e62c2873c86bf
2024-11-08T00:43:57,311 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/ns/2d6f1171b33448c1991611a1e72d1495 is 106, key is testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b./ns:/1731026636671/DeleteFamily/seqid=0
2024-11-08T00:43:57,316 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741845_1021 (size=6009)
2024-11-08T00:43:57,371 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:57,571 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:57,718 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/ns/2d6f1171b33448c1991611a1e72d1495
2024-11-08T00:43:57,745 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/rep_barrier/e073afa4a3d74040a7f5cdacc4711bbd is 115, key is testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b./rep_barrier:/1731026636671/DeleteFamily/seqid=0
2024-11-08T00:43:57,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741846_1022 (size=6121)
2024-11-08T00:43:57,772 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:57,972 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:58,151 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/rep_barrier/e073afa4a3d74040a7f5cdacc4711bbd
2024-11-08T00:43:58,172 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:58,176 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/table/fe61ca2222da407fab54eaa4ca30d7b1 is 109, key is testRecoveryAndDoubleExecution,a,1731026629246.24d35a011749709cd51a9cfe14e19e0b./table:/1731026636671/DeleteFamily/seqid=0
2024-11-08T00:43:58,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741847_1023 (size=6158)
2024-11-08T00:43:58,373 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:58,573 DEBUG [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T00:43:58,582 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/table/fe61ca2222da407fab54eaa4ca30d7b1
2024-11-08T00:43:58,591 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/info/3c6c493024c441ecac2e62c2873c86bf as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/info/3c6c493024c441ecac2e62c2873c86bf
2024-11-08T00:43:58,599 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/info/3c6c493024c441ecac2e62c2873c86bf, entries=12, sequenceid=78, filesize=6.5 K
2024-11-08T00:43:58,601 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/ns/2d6f1171b33448c1991611a1e72d1495 as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/ns/2d6f1171b33448c1991611a1e72d1495
2024-11-08T00:43:58,608 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/ns/2d6f1171b33448c1991611a1e72d1495, entries=8, sequenceid=78, filesize=5.9 K
2024-11-08T00:43:58,609 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/rep_barrier/e073afa4a3d74040a7f5cdacc4711bbd as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/rep_barrier/e073afa4a3d74040a7f5cdacc4711bbd
2024-11-08T00:43:58,618 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/rep_barrier/e073afa4a3d74040a7f5cdacc4711bbd, entries=6, sequenceid=78, filesize=6.0 K
2024-11-08T00:43:58,619 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/.tmp/table/fe61ca2222da407fab54eaa4ca30d7b1 as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/table/fe61ca2222da407fab54eaa4ca30d7b1
2024-11-08T00:43:58,627 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/table/fe61ca2222da407fab54eaa4ca30d7b1, entries=9, sequenceid=78, filesize=6.0 K
2024-11-08T00:43:58,633 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~25.47 KB/26078, heapSize ~41.58 KB/42576, currentSize=0 B/0 for 1588230740 in 1858ms, sequenceid=78, compaction requested=false
2024-11-08T00:43:58,641 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/data/hbase/meta/1588230740/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1
2024-11-08T00:43:58,642 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-08T00:43:58,642 INFO  [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T00:43:58,642 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731026636770Running coprocessor pre-close hooks at 1731026636770Disabling compacts and flushes for region at 1731026636770Disabling writes for close at 1731026636771 (+1 ms)Obtaining lock to block concurrent updates at 1731026636774 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1731026636774Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=26078, getHeapSize=42576, getOffHeapSize=0, getCellsCount=205 at 1731026636782 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731026636783 (+1 ms)Flushing 1588230740/info: creating writer at 1731026636786 (+3 ms)Flushing 1588230740/info: appending metadata at 1731026636829 (+43 ms)Flushing 1588230740/info: closing flushed file at 1731026636832 (+3 ms)Flushing 1588230740/ns: creating writer at 1731026637294 (+462 ms)Flushing 1588230740/ns: appending metadata at 1731026637310 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731026637310Flushing 1588230740/rep_barrier: creating writer at 1731026637727 (+417 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731026637744 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731026637744Flushing 1588230740/table: creating writer at 1731026638160 (+416 ms)Flushing 1588230740/table: appending metadata at 1731026638175 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731026638175Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36049059: reopening flushed file at 1731026638590 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@377b7083: reopening flushed file at 1731026638600 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a678bf8: reopening flushed file at 1731026638608 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d70908: reopening flushed file at 1731026638618 (+10 ms)Finished flush of dataSize ~25.47 KB/26078, heapSize ~41.58 KB/42576, currentSize=0 B/0 for 1588230740 in 1858ms, sequenceid=78, compaction requested=false at 1731026638633 (+15 ms)Writing region close event to WAL at 1731026638636 (+3 ms)Running coprocessor post-close hooks at 1731026638641 (+5 ms)Closed at 1731026638642 (+1 ms)
2024-11-08T00:43:58,642 DEBUG [RS_CLOSE_META-regionserver/c302dddf724e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-08T00:43:58,773 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(976): stopping server c302dddf724e,46797,1731026622161; all regions closed.
2024-11-08T00:43:58,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741834_1010 (size=30662)
2024-11-08T00:43:58,782 DEBUG [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/oldWALs
2024-11-08T00:43:58,782 INFO  [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C46797%2C1731026622161.meta:.meta(num 1731026625532)
2024-11-08T00:43:58,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741833_1009 (size=4131)
2024-11-08T00:43:58,786 DEBUG [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/oldWALs
2024-11-08T00:43:58,786 INFO  [RS:0;c302dddf724e:46797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c302dddf724e%2C46797%2C1731026622161:(num 1731026625172)
2024-11-08T00:43:58,787 DEBUG [RS:0;c302dddf724e:46797 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T00:43:58,787 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T00:43:58,787 INFO  [RS:0;c302dddf724e:46797 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T00:43:58,787 INFO  [RS:0;c302dddf724e:46797 {}] hbase.ChoreService(370): Chore service for: regionserver/c302dddf724e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-08T00:43:58,787 INFO  [RS:0;c302dddf724e:46797 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T00:43:58,787 INFO  [regionserver/c302dddf724e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T00:43:58,787 INFO  [RS:0;c302dddf724e:46797 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46797
2024-11-08T00:43:58,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T00:43:58,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c302dddf724e,46797,1731026622161
2024-11-08T00:43:58,791 INFO  [RS:0;c302dddf724e:46797 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T00:43:58,792 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c302dddf724e,46797,1731026622161]
2024-11-08T00:43:58,794 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c302dddf724e,46797,1731026622161 already deleted, retry=false
2024-11-08T00:43:58,794 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c302dddf724e,46797,1731026622161 expired; onlineServers=0
2024-11-08T00:43:58,794 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c302dddf724e,44077,1731026621470' *****
2024-11-08T00:43:58,794 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-08T00:43:58,794 INFO  [M:0;c302dddf724e:44077 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T00:43:58,795 INFO  [M:0;c302dddf724e:44077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T00:43:58,795 DEBUG [M:0;c302dddf724e:44077 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-08T00:43:58,795 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-08T00:43:58,795 DEBUG [M:0;c302dddf724e:44077 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-08T00:43:58,795 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026624809 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.large.0-1731026624809,5,FailOnTimeoutGroup]
2024-11-08T00:43:58,795 DEBUG [master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026624810 {}] cleaner.HFileCleaner(306): Exit Thread[master/c302dddf724e:0:becomeActiveMaster-HFileCleaner.small.0-1731026624810,5,FailOnTimeoutGroup]
2024-11-08T00:43:58,795 INFO  [M:0;c302dddf724e:44077 {}] hbase.ChoreService(370): Chore service for: master/c302dddf724e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-08T00:43:58,795 INFO  [M:0;c302dddf724e:44077 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T00:43:58,795 DEBUG [M:0;c302dddf724e:44077 {}] master.HMaster(1795): Stopping service threads
2024-11-08T00:43:58,795 INFO  [M:0;c302dddf724e:44077 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-08T00:43:58,796 INFO  [M:0;c302dddf724e:44077 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T00:43:58,796 ERROR [M:0;c302dddf724e:44077 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-3,5,PEWorkerGroup]
    Thread[HFileArchiver-4,5,PEWorkerGroup]
    Thread[HFileArchiver-5,5,PEWorkerGroup]
    Thread[HFileArchiver-6,5,PEWorkerGroup]
2024-11-08T00:43:58,796 INFO  [M:0;c302dddf724e:44077 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T00:43:58,796 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-08T00:43:58,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T00:43:58,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T00:43:58,798 DEBUG [M:0;c302dddf724e:44077 {}] zookeeper.ZKUtil(347): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-08T00:43:58,798 WARN  [M:0;c302dddf724e:44077 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-08T00:43:58,799 INFO  [M:0;c302dddf724e:44077 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/.lastflushedseqids
2024-11-08T00:43:58,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741848_1024 (size=72)
2024-11-08T00:43:58,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T00:43:58,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46797-0x100f507fb490001, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T00:43:58,894 INFO  [RS:0;c302dddf724e:46797 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T00:43:58,895 INFO  [RS:0;c302dddf724e:46797 {}] regionserver.HRegionServer(1031): Exiting; stopping=c302dddf724e,46797,1731026622161; zookeeper connection closed.
2024-11-08T00:43:58,895 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@614974d8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@614974d8
2024-11-08T00:43:58,895 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-08T00:43:59,210 INFO  [M:0;c302dddf724e:44077 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T00:43:59,211 INFO  [M:0;c302dddf724e:44077 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-08T00:43:59,211 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T00:43:59,211 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:59,211 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:59,211 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T00:43:59,211 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:43:59,211 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=149.09 KB heapSize=178.88 KB
2024-11-08T00:43:59,229 DEBUG [M:0;c302dddf724e:44077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0312eb388a54cd7810bf77270ed8f14 is 82, key is hbase:meta,,1/info:regioninfo/1731026625622/Put/seqid=0
2024-11-08T00:43:59,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741849_1025 (size=5672)
2024-11-08T00:43:59,634 INFO  [M:0;c302dddf724e:44077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0312eb388a54cd7810bf77270ed8f14
2024-11-08T00:43:59,662 DEBUG [M:0;c302dddf724e:44077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6c80c8fcfc245fea45f9459388ebec0 is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1731026630526/Put/seqid=0
2024-11-08T00:43:59,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741850_1026 (size=11517)
2024-11-08T00:44:00,068 INFO  [M:0;c302dddf724e:44077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6c80c8fcfc245fea45f9459388ebec0
2024-11-08T00:44:00,094 DEBUG [M:0;c302dddf724e:44077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17f04fb074074dbd8702727239bd3d92 is 69, key is c302dddf724e,46797,1731026622161/rs:state/1731026624941/Put/seqid=0
2024-11-08T00:44:00,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741851_1027 (size=5156)
2024-11-08T00:44:00,500 INFO  [M:0;c302dddf724e:44077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17f04fb074074dbd8702727239bd3d92
2024-11-08T00:44:00,508 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0312eb388a54cd7810bf77270ed8f14 as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d0312eb388a54cd7810bf77270ed8f14
2024-11-08T00:44:00,515 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d0312eb388a54cd7810bf77270ed8f14, entries=8, sequenceid=328, filesize=5.5 K
2024-11-08T00:44:00,516 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6c80c8fcfc245fea45f9459388ebec0 as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b6c80c8fcfc245fea45f9459388ebec0
2024-11-08T00:44:00,524 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b6c80c8fcfc245fea45f9459388ebec0, entries=39, sequenceid=328, filesize=11.2 K
2024-11-08T00:44:00,525 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/17f04fb074074dbd8702727239bd3d92 as hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/17f04fb074074dbd8702727239bd3d92
2024-11-08T00:44:00,531 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44739/user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/17f04fb074074dbd8702727239bd3d92, entries=1, sequenceid=328, filesize=5.0 K
2024-11-08T00:44:00,533 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.09 KB/152670, heapSize ~178.58 KB/182864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=328, compaction requested=false
2024-11-08T00:44:00,534 INFO  [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T00:44:00,534 DEBUG [M:0;c302dddf724e:44077 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731026639211Disabling compacts and flushes for region at 1731026639211Disabling writes for close at 1731026639211Obtaining lock to block concurrent updates at 1731026639211Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731026639211Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=152670, getHeapSize=183104, getOffHeapSize=0, getCellsCount=389 at 1731026639211Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731026639212 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731026639212Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731026639228 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731026639228Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731026639641 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731026639662 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731026639662Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731026640077 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731026640094 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731026640094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a94055c: reopening flushed file at 1731026640507 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35b8a14: reopening flushed file at 1731026640515 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f180e02: reopening flushed file at 1731026640524 (+9 ms)Finished flush of dataSize ~149.09 KB/152670, heapSize ~178.58 KB/182864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=328, compaction requested=false at 1731026640533 (+9 ms)Writing region close event to WAL at 1731026640534 (+1 ms)Closed at 1731026640534
2024-11-08T00:44:00,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44031 is added to blk_1073741830_1006 (size=174653)
2024-11-08T00:44:00,537 WARN  [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5ecbbc28-252c-fee5-5323-8137d14e0269/MasterData/WALs/c302dddf724e,44077,1731026621470/c302dddf724e%2C44077%2C1731026621470.1731026624262 not finished, retry = 0
2024-11-08T00:44:00,639 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T00:44:00,639 INFO  [M:0;c302dddf724e:44077 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-08T00:44:00,639 INFO  [M:0;c302dddf724e:44077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44077
2024-11-08T00:44:00,640 INFO  [M:0;c302dddf724e:44077 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T00:44:00,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T00:44:00,742 INFO  [M:0;c302dddf724e:44077 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T00:44:00,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44077-0x100f507fb490000, quorum=127.0.0.1:53666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T00:44:00,746 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156ffeaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T00:44:00,749 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T00:44:00,749 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T00:44:00,749 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T00:44:00,749 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.log.dir/,STOPPED}
2024-11-08T00:44:00,751 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T00:44:00,751 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T00:44:00,751 WARN  [BP-1589806164-172.17.0.2-1731026618582 heartbeating to localhost/127.0.0.1:44739 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T00:44:00,752 WARN  [BP-1589806164-172.17.0.2-1731026618582 heartbeating to localhost/127.0.0.1:44739 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1589806164-172.17.0.2-1731026618582 (Datanode Uuid 48d57e7a-9ae2-45e9-b239-10bcdf31a27b) service to localhost/127.0.0.1:44739
2024-11-08T00:44:00,753 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/data/data1/current/BP-1589806164-172.17.0.2-1731026618582 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T00:44:00,753 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/cluster_7f613979-358d-1b2e-9a7d-ced9f5cf3c8e/data/data2/current/BP-1589806164-172.17.0.2-1731026618582 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T00:44:00,754 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T00:44:00,762 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27b19a13{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T00:44:00,763 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T00:44:00,763 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T00:44:00,763 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T00:44:00,763 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c122a767-7448-a76c-c046-cc61683ead90/hadoop.log.dir/,STOPPED}
2024-11-08T00:44:00,770 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-08T00:44:00,787 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down