2024-11-11 16:14:48,781 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-11 16:14:48,825 main DEBUG Took 0.040975 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 16:14:48,826 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 16:14:48,827 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 16:14:48,828 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 16:14:48,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,858 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 16:14:48,901 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,905 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,907 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,908 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,909 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,910 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,911 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,912 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,919 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,924 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,927 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,928 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:48,928 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,929 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 16:14:48,955 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:48,962 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 16:14:48,964 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 16:14:48,965 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 16:14:48,967 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 16:14:48,967 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 16:14:49,005 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 16:14:49,022 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 16:14:49,028 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 16:14:49,032 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 16:14:49,033 main DEBUG createAppenders(={Console}) 2024-11-11 16:14:49,034 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-11 16:14:49,034 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-11 16:14:49,035 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-11 16:14:49,038 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 16:14:49,039 main DEBUG OutputStream closed 2024-11-11 16:14:49,040 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 16:14:49,041 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 16:14:49,041 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-11 16:14:49,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 16:14:49,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 16:14:49,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 16:14:49,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 16:14:49,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 16:14:49,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 16:14:49,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 16:14:49,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 16:14:49,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 16:14:49,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 16:14:49,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 16:14:49,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 16:14:49,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 16:14:49,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 16:14:49,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 16:14:49,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 16:14:49,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 16:14:49,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 16:14:49,220 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 16:14:49,221 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-11 16:14:49,221 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 16:14:49,222 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-11T16:14:49,242 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-11 16:14:49,263 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 16:14:49,264 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T16:14:49,823 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7 2024-11-11T16:14:49,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T16:14:49,912 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e, deleteOnExit=true 2024-11-11T16:14:49,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T16:14:49,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/test.cache.data in system properties and HBase conf 2024-11-11T16:14:49,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T16:14:49,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.log.dir in system properties and HBase conf 2024-11-11T16:14:49,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T16:14:49,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T16:14:49,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T16:14:50,157 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T16:14:50,395 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T16:14:50,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T16:14:50,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T16:14:50,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T16:14:50,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T16:14:50,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T16:14:50,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T16:14:50,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T16:14:50,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T16:14:50,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T16:14:50,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/nfs.dump.dir in system properties and HBase conf 2024-11-11T16:14:50,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/java.io.tmpdir in system properties and HBase conf 2024-11-11T16:14:50,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T16:14:50,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T16:14:50,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T16:14:52,000 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T16:14:52,099 INFO [Time-limited test {}] log.Log(170): Logging initialized @4755ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T16:14:52,218 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:52,315 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:52,419 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:52,420 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:52,425 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T16:14:52,454 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:52,459 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:52,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:52,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3260f08c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/java.io.tmpdir/jetty-localhost-36221-hadoop-hdfs-3_4_1-tests_jar-_-any-8686855831202270871/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T16:14:52,912 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:36221} 2024-11-11T16:14:52,912 INFO [Time-limited test {}] server.Server(415): Started @5570ms 2024-11-11T16:14:53,679 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:53,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:53,701 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:53,701 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:53,701 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T16:14:53,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c81ef04{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:53,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cf3b4f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:53,886 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b9032ec{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/java.io.tmpdir/jetty-localhost-33555-hadoop-hdfs-3_4_1-tests_jar-_-any-5511554730518216569/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T16:14:53,887 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a83a39b{HTTP/1.1, (http/1.1)}{localhost:33555} 2024-11-11T16:14:53,887 INFO [Time-limited test {}] server.Server(415): Started @6545ms 2024-11-11T16:14:53,968 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T16:14:54,311 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:54,325 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:54,345 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:54,345 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:54,346 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T16:14:54,347 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20976991{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:54,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c35a52f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:54,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bfd0738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/java.io.tmpdir/jetty-localhost-41715-hadoop-hdfs-3_4_1-tests_jar-_-any-6632404984303056826/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T16:14:54,542 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@800a6b{HTTP/1.1, (http/1.1)}{localhost:41715} 2024-11-11T16:14:54,542 INFO [Time-limited test {}] server.Server(415): Started @7200ms 2024-11-11T16:14:54,545 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T16:14:54,792 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:54,817 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:54,838 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data1/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:54,852 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:54,852 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:54,853 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T16:14:54,853 WARN [Thread-102 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data2/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:54,869 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@370e587d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:54,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@792fda78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:54,901 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data3/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:54,945 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data4/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:55,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25cc6d31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/java.io.tmpdir/jetty-localhost-39665-hadoop-hdfs-3_4_1-tests_jar-_-any-8647630685748690877/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T16:14:55,083 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b45c75d{HTTP/1.1, (http/1.1)}{localhost:39665} 2024-11-11T16:14:55,083 INFO [Time-limited test {}] server.Server(415): Started @7741ms 2024-11-11T16:14:55,087 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T16:14:55,097 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T16:14:55,112 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T16:14:55,240 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b121dd22b8a69 with lease ID 0x33463d9386555b64: Processing first storage report for DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce from datanode DatanodeRegistration(127.0.0.1:45315, datanodeUuid=1b5f970b-b00f-487a-94cf-e5ba8d092014, infoPort=41647, infoSecurePort=0, ipcPort=45527, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b121dd22b8a69 with lease ID 0x33463d9386555b64: from storage DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce node DatanodeRegistration(127.0.0.1:45315, datanodeUuid=1b5f970b-b00f-487a-94cf-e5ba8d092014, infoPort=41647, infoSecurePort=0, ipcPort=45527, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b121dd22b8a69 with lease ID 0x33463d9386555b64: Processing first storage report for DS-de8da307-34e4-43c3-969d-a99a0d110bee from datanode DatanodeRegistration(127.0.0.1:45315, datanodeUuid=1b5f970b-b00f-487a-94cf-e5ba8d092014, infoPort=41647, infoSecurePort=0, ipcPort=45527, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,243 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b121dd22b8a69 with lease ID 0x33463d9386555b64: from storage DS-de8da307-34e4-43c3-969d-a99a0d110bee node DatanodeRegistration(127.0.0.1:45315, datanodeUuid=1b5f970b-b00f-487a-94cf-e5ba8d092014, infoPort=41647, infoSecurePort=0, ipcPort=45527, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8a8dcbd8ab509d5 with lease ID 0x33463d9386555b65: Processing first storage report for DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2 from datanode DatanodeRegistration(127.0.0.1:40759, datanodeUuid=07a04e14-2da6-4f10-a2a7-05da805c27b2, infoPort=33195, infoSecurePort=0, ipcPort=44331, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8a8dcbd8ab509d5 with lease ID 0x33463d9386555b65: from storage DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2 node DatanodeRegistration(127.0.0.1:40759, datanodeUuid=07a04e14-2da6-4f10-a2a7-05da805c27b2, infoPort=33195, infoSecurePort=0, ipcPort=44331, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8a8dcbd8ab509d5 with lease ID 0x33463d9386555b65: Processing first storage report for DS-412e67ab-d83b-4d6f-b48c-9733bf3b2c2a from datanode DatanodeRegistration(127.0.0.1:40759, datanodeUuid=07a04e14-2da6-4f10-a2a7-05da805c27b2, infoPort=33195, infoSecurePort=0, ipcPort=44331, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,248 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8a8dcbd8ab509d5 with lease ID 0x33463d9386555b65: from storage DS-412e67ab-d83b-4d6f-b48c-9733bf3b2c2a node DatanodeRegistration(127.0.0.1:40759, datanodeUuid=07a04e14-2da6-4f10-a2a7-05da805c27b2, infoPort=33195, infoSecurePort=0, ipcPort=44331, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,506 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data5/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:55,521 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/data/data6/current/BP-653416662-172.17.0.2-1731341691652/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:55,646 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T16:14:55,659 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdfde2bd5c142c72e with lease ID 0x33463d9386555b66: Processing first storage report for DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c from datanode DatanodeRegistration(127.0.0.1:37169, datanodeUuid=3b67d8b9-e371-4a14-abbb-1e65270fab1c, infoPort=33177, infoSecurePort=0, ipcPort=43949, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,659 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdfde2bd5c142c72e with lease ID 0x33463d9386555b66: from storage DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c node DatanodeRegistration(127.0.0.1:37169, datanodeUuid=3b67d8b9-e371-4a14-abbb-1e65270fab1c, infoPort=33177, infoSecurePort=0, ipcPort=43949, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,660 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdfde2bd5c142c72e with lease ID 0x33463d9386555b66: Processing first storage report for DS-3f822ca7-cf6d-4ef9-8b83-a89aeff6cf9e from datanode DatanodeRegistration(127.0.0.1:37169, datanodeUuid=3b67d8b9-e371-4a14-abbb-1e65270fab1c, infoPort=33177, infoSecurePort=0, ipcPort=43949, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652) 2024-11-11T16:14:55,660 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdfde2bd5c142c72e with lease ID 0x33463d9386555b66: from storage DS-3f822ca7-cf6d-4ef9-8b83-a89aeff6cf9e node DatanodeRegistration(127.0.0.1:37169, datanodeUuid=3b67d8b9-e371-4a14-abbb-1e65270fab1c, infoPort=33177, infoSecurePort=0, ipcPort=43949, storageInfo=lv=-57;cid=testClusterID;nsid=980740944;c=1731341691652), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T16:14:55,954 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7 2024-11-11T16:14:56,162 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/zookeeper_0, clientPort=54959, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51df5ebe-43d4-e1c4-47d7-9178ddf650f7/cluster_54900d3f-89c1-f55e-b61d-53bce29e9c3e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T16:14:56,192 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54959 2024-11-11T16:14:56,211 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:56,217 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:56,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741825_1001 (size=7) 2024-11-11T16:14:56,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741825_1001 (size=7) 2024-11-11T16:14:56,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741825_1001 (size=7) 2024-11-11T16:14:57,060 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba with version=8 2024-11-11T16:14:57,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/hbase-staging 2024-11-11T16:14:57,264 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T16:14:57,668 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:57,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:57,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:57,696 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:57,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:57,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:57,916 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T16:14:57,993 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T16:14:58,007 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T16:14:58,013 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:58,054 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3132 (auto-detected) 2024-11-11T16:14:58,056 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T16:14:58,090 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35895 2024-11-11T16:14:58,124 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35895 connecting to ZooKeeper ensemble=127.0.0.1:54959 2024-11-11T16:14:58,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358950x0, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:58,195 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35895-0x1002f9fc70b0000 connected 2024-11-11T16:14:58,301 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,304 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,318 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:58,324 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba, hbase.cluster.distributed=false 2024-11-11T16:14:58,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:58,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35895 2024-11-11T16:14:58,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35895 2024-11-11T16:14:58,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35895 2024-11-11T16:14:58,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35895 2024-11-11T16:14:58,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35895 2024-11-11T16:14:58,551 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:58,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,554 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:58,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:58,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T16:14:58,562 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:58,568 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44579 2024-11-11T16:14:58,578 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44579 connecting to ZooKeeper ensemble=127.0.0.1:54959 2024-11-11T16:14:58,582 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445790x0, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:58,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44579-0x1002f9fc70b0001 connected 2024-11-11T16:14:58,637 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:58,643 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T16:14:58,659 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T16:14:58,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T16:14:58,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:58,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44579 2024-11-11T16:14:58,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44579 2024-11-11T16:14:58,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44579 2024-11-11T16:14:58,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44579 2024-11-11T16:14:58,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44579 2024-11-11T16:14:58,716 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:58,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,718 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:58,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:58,719 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T16:14:58,720 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:58,721 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33497 2024-11-11T16:14:58,724 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33497 connecting to ZooKeeper ensemble=127.0.0.1:54959 2024-11-11T16:14:58,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,731 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334970x0, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:58,761 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334970x0, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:58,761 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T16:14:58,765 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33497-0x1002f9fc70b0002 connected 2024-11-11T16:14:58,780 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T16:14:58,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T16:14:58,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:58,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33497 2024-11-11T16:14:58,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33497 2024-11-11T16:14:58,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33497 2024-11-11T16:14:58,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33497 2024-11-11T16:14:58,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33497 2024-11-11T16:14:58,839 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:58,841 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,841 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,842 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:58,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:58,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:58,842 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T16:14:58,845 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:58,853 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36245 2024-11-11T16:14:58,855 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36245 connecting to ZooKeeper ensemble=127.0.0.1:54959 2024-11-11T16:14:58,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,864 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:58,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362450x0, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:58,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:362450x0, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:58,880 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T16:14:58,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36245-0x1002f9fc70b0003 connected 2024-11-11T16:14:58,888 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T16:14:58,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T16:14:58,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:58,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36245 2024-11-11T16:14:58,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36245 2024-11-11T16:14:58,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36245 2024-11-11T16:14:58,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36245 2024-11-11T16:14:58,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36245 2024-11-11T16:14:58,930 DEBUG [M:0;16b413a53992:35895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16b413a53992:35895 2024-11-11T16:14:58,942 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16b413a53992,35895,1731341697364 2024-11-11T16:14:58,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:58,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:58,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:58,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:58,963 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16b413a53992,35895,1731341697364 2024-11-11T16:14:58,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T16:14:58,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T16:14:58,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:58,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:58,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T16:14:58,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:58,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:59,004 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T16:14:59,005 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16b413a53992,35895,1731341697364 from backup master directory 2024-11-11T16:14:59,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16b413a53992,35895,1731341697364 2024-11-11T16:14:59,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:59,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:59,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:59,011 WARN [master/16b413a53992:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:14:59,012 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16b413a53992,35895,1731341697364 2024-11-11T16:14:59,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:59,017 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T16:14:59,021 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T16:14:59,147 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/hbase.id] with ID: 77267fe1-1280-4bf0-aa3a-d386e646f2df 2024-11-11T16:14:59,147 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/.tmp/hbase.id 2024-11-11T16:14:59,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741826_1002 (size=42) 2024-11-11T16:14:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741826_1002 (size=42) 2024-11-11T16:14:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741826_1002 (size=42) 2024-11-11T16:14:59,178 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/.tmp/hbase.id]:[hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/hbase.id] 2024-11-11T16:14:59,299 INFO [master/16b413a53992:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:59,310 INFO [master/16b413a53992:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T16:14:59,357 INFO [master/16b413a53992:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 38ms. 2024-11-11T16:14:59,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741827_1003 (size=196) 2024-11-11T16:14:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741827_1003 (size=196) 2024-11-11T16:14:59,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741827_1003 (size=196) 2024-11-11T16:14:59,473 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:14:59,477 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T16:14:59,502 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,508 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:14:59,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741828_1004 (size=1189) 2024-11-11T16:14:59,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741828_1004 (size=1189) 2024-11-11T16:14:59,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741828_1004 (size=1189) 2024-11-11T16:14:59,615 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store 2024-11-11T16:14:59,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741829_1005 (size=34) 2024-11-11T16:14:59,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741829_1005 (size=34) 2024-11-11T16:14:59,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741829_1005 (size=34) 2024-11-11T16:14:59,678 INFO [master/16b413a53992:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T16:14:59,683 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:59,685 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T16:14:59,685 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:59,685 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:59,688 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T16:14:59,688 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:59,688 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:59,689 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731341699684Disabling compacts and flushes for region at 1731341699684Disabling writes for close at 1731341699688 (+4 ms)Writing region close event to WAL at 1731341699688Closed at 1731341699688 2024-11-11T16:14:59,692 WARN [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/.initializing 2024-11-11T16:14:59,693 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/WALs/16b413a53992,35895,1731341697364 2024-11-11T16:14:59,704 INFO [master/16b413a53992:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:14:59,728 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C35895%2C1731341697364, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/WALs/16b413a53992,35895,1731341697364, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/oldWALs, maxLogs=10 2024-11-11T16:14:59,761 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/WALs/16b413a53992,35895,1731341697364/16b413a53992%2C35895%2C1731341697364.1731341699737, exclude list is [], retry=0 2024-11-11T16:14:59,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:14:59,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:14:59,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:14:59,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T16:14:59,848 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/WALs/16b413a53992,35895,1731341697364/16b413a53992%2C35895%2C1731341697364.1731341699737 2024-11-11T16:14:59,849 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33177:33177),(127.0.0.1/127.0.0.1:41647:41647),(127.0.0.1/127.0.0.1:33195:33195)] 2024-11-11T16:14:59,850 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:59,851 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:59,855 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,856 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T16:14:59,957 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:59,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:59,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T16:14:59,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:59,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:59,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T16:14:59,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:59,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:59,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:59,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T16:14:59,995 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:59,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:59,998 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,006 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,009 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,017 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,018 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,023 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T16:15:00,029 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:15:00,035 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:00,038 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67563517, jitterRate=0.0067748576402664185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T16:15:00,048 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731341699874Initializing all the Stores at 1731341699877 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341699878 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341699880 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341699880Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341699881 (+1 ms)Cleaning up temporary data from old regions at 1731341700018 (+137 ms)Region opened successfully at 1731341700048 (+30 ms) 2024-11-11T16:15:00,053 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T16:15:00,103 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b065e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:15:00,153 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T16:15:00,170 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:00,171 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:00,175 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:00,177 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-11T16:15:00,184 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-11T16:15:00,185 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T16:15:00,229 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T16:15:00,241 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T16:15:00,243 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T16:15:00,247 INFO [master/16b413a53992:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T16:15:00,249 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T16:15:00,251 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T16:15:00,253 INFO [master/16b413a53992:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T16:15:00,257 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T16:15:00,262 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T16:15:00,263 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T16:15:00,265 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T16:15:00,289 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T16:15:00,291 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T16:15:00,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:00,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:00,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:00,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:00,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,302 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16b413a53992,35895,1731341697364, sessionid=0x1002f9fc70b0000, setting cluster-up flag (Was=false) 2024-11-11T16:15:00,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,327 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T16:15:00,329 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b413a53992,35895,1731341697364 2024-11-11T16:15:00,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:00,343 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T16:15:00,345 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b413a53992,35895,1731341697364 2024-11-11T16:15:00,352 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T16:15:00,412 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(746): ClusterId : 77267fe1-1280-4bf0-aa3a-d386e646f2df 2024-11-11T16:15:00,413 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(746): ClusterId : 77267fe1-1280-4bf0-aa3a-d386e646f2df 2024-11-11T16:15:00,416 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T16:15:00,416 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T16:15:00,425 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T16:15:00,426 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T16:15:00,426 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(746): ClusterId : 77267fe1-1280-4bf0-aa3a-d386e646f2df 2024-11-11T16:15:00,426 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T16:15:00,428 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T16:15:00,428 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T16:15:00,433 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T16:15:00,434 DEBUG [RS:1;16b413a53992:33497 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37100f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:15:00,434 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T16:15:00,434 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T16:15:00,447 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T16:15:00,448 DEBUG [RS:0;16b413a53992:44579 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ce9bc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:15:00,452 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T16:15:00,452 DEBUG [RS:2;16b413a53992:36245 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5a7e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:15:00,461 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:00,473 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16b413a53992:44579 2024-11-11T16:15:00,477 INFO [RS:0;16b413a53992:44579 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T16:15:00,478 INFO [RS:0;16b413a53992:44579 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T16:15:00,478 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T16:15:00,491 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;16b413a53992:36245 2024-11-11T16:15:00,491 INFO [RS:2;16b413a53992:36245 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T16:15:00,491 INFO [RS:2;16b413a53992:36245 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T16:15:00,491 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T16:15:00,491 INFO [master/16b413a53992:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T16:15:00,494 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,35895,1731341697364 with port=36245, startcode=1731341698838 2024-11-11T16:15:00,494 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,35895,1731341697364 with port=44579, startcode=1731341698497 2024-11-11T16:15:00,494 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;16b413a53992:33497 2024-11-11T16:15:00,497 INFO [RS:1;16b413a53992:33497 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T16:15:00,497 INFO [RS:1;16b413a53992:33497 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T16:15:00,497 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T16:15:00,499 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,35895,1731341697364 with port=33497, startcode=1731341698716 2024-11-11T16:15:00,512 DEBUG [RS:2;16b413a53992:36245 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T16:15:00,512 DEBUG [RS:1;16b413a53992:33497 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T16:15:00,514 DEBUG [RS:0;16b413a53992:44579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T16:15:00,516 INFO [master/16b413a53992:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T16:15:00,529 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16b413a53992,35895,1731341697364 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T16:15:00,541 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:15:00,541 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:15:00,541 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:15:00,541 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:15:00,542 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16b413a53992:0, corePoolSize=10, maxPoolSize=10 2024-11-11T16:15:00,542 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,542 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:15:00,542 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,561 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341730561 2024-11-11T16:15:00,563 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T16:15:00,566 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T16:15:00,572 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T16:15:00,572 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T16:15:00,573 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T16:15:00,573 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T16:15:00,586 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:00,587 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T16:15:00,595 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,603 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T16:15:00,604 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:00,604 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T16:15:00,605 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T16:15:00,605 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T16:15:00,625 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T16:15:00,625 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T16:15:00,626 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T16:15:00,628 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37065, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T16:15:00,629 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34635, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T16:15:00,633 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,643 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,657 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341700627,5,FailOnTimeoutGroup] 2024-11-11T16:15:00,658 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341700657,5,FailOnTimeoutGroup] 2024-11-11T16:15:00,658 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,659 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T16:15:00,660 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,661 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b413a53992,44579,1731341698497 2024-11-11T16:15:00,674 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(517): Registering regionserver=16b413a53992,44579,1731341698497 2024-11-11T16:15:00,689 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:00,689 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34433 2024-11-11T16:15:00,689 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T16:15:00,692 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T16:15:00,692 WARN [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T16:15:00,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:15:00,697 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T16:15:00,697 WARN [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T16:15:00,704 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b413a53992,44579,1731341698497] 2024-11-11T16:15:00,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741831_1007 (size=1321) 2024-11-11T16:15:00,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741831_1007 (size=1321) 2024-11-11T16:15:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741831_1007 (size=1321) 2024-11-11T16:15:00,710 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T16:15:00,711 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:00,712 DEBUG [RS:0;16b413a53992:44579 {}] zookeeper.ZKUtil(111): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b413a53992,44579,1731341698497 2024-11-11T16:15:00,712 WARN [RS:0;16b413a53992:44579 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:15:00,712 INFO [RS:0;16b413a53992:44579 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:15:00,712 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,44579,1731341698497 2024-11-11T16:15:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741832_1008 (size=32) 2024-11-11T16:15:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741832_1008 (size=32) 2024-11-11T16:15:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741832_1008 (size=32) 2024-11-11T16:15:00,764 INFO [RS:0;16b413a53992:44579 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T16:15:00,794 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,35895,1731341697364 with port=33497, startcode=1731341698716 2024-11-11T16:15:00,796 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b413a53992,33497,1731341698716 2024-11-11T16:15:00,796 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(517): Registering regionserver=16b413a53992,33497,1731341698716 2024-11-11T16:15:00,798 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,35895,1731341697364 with port=36245, startcode=1731341698838 2024-11-11T16:15:00,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b413a53992,36245,1731341698838 2024-11-11T16:15:00,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] master.ServerManager(517): Registering regionserver=16b413a53992,36245,1731341698838 2024-11-11T16:15:00,805 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:00,805 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34433 2024-11-11T16:15:00,805 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T16:15:00,809 INFO [RS:0;16b413a53992:44579 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T16:15:00,810 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:00,810 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34433 2024-11-11T16:15:00,810 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T16:15:00,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:15:00,817 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b413a53992,33497,1731341698716] 2024-11-11T16:15:00,817 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b413a53992,36245,1731341698838] 2024-11-11T16:15:00,817 DEBUG [RS:2;16b413a53992:36245 {}] zookeeper.ZKUtil(111): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b413a53992,36245,1731341698838 2024-11-11T16:15:00,817 WARN [RS:2;16b413a53992:36245 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:15:00,817 INFO [RS:2;16b413a53992:36245 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:15:00,817 DEBUG [RS:1;16b413a53992:33497 {}] zookeeper.ZKUtil(111): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b413a53992,33497,1731341698716 2024-11-11T16:15:00,817 DEBUG [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838 2024-11-11T16:15:00,818 WARN [RS:1;16b413a53992:33497 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:15:00,818 INFO [RS:1;16b413a53992:33497 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:15:00,818 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716 2024-11-11T16:15:00,822 INFO [RS:0;16b413a53992:44579 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T16:15:00,822 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,824 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T16:15:00,832 INFO [RS:0;16b413a53992:44579 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T16:15:00,835 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,835 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,836 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,836 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,836 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,836 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,837 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:15:00,837 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,837 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,837 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,837 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,838 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,838 INFO [RS:1;16b413a53992:33497 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T16:15:00,838 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,838 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,838 DEBUG [RS:0;16b413a53992:44579 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,849 INFO [RS:1;16b413a53992:33497 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T16:15:00,852 INFO [RS:2;16b413a53992:36245 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T16:15:00,869 INFO [RS:1;16b413a53992:33497 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T16:15:00,869 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,881 INFO [RS:2;16b413a53992:36245 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T16:15:00,881 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,881 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,881 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,881 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,881 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,882 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,44579,1731341698497-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:15:00,884 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T16:15:00,885 INFO [RS:2;16b413a53992:36245 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T16:15:00,885 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,888 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T16:15:00,892 INFO [RS:2;16b413a53992:36245 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T16:15:00,892 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,893 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,893 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,894 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,894 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,894 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,894 INFO [RS:1;16b413a53992:33497 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T16:15:00,894 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:15:00,894 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,895 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,895 DEBUG [RS:2;16b413a53992:36245 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,896 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,897 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:15:00,897 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,897 DEBUG [RS:1;16b413a53992:33497 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:15:00,900 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,36245,1731341698838-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,901 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33497,1731341698716-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:15:00,912 INFO [RS:0;16b413a53992:44579 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T16:15:00,915 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,44579,1731341698497-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,916 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,916 INFO [RS:0;16b413a53992:44579 {}] regionserver.Replication(171): 16b413a53992,44579,1731341698497 started 2024-11-11T16:15:00,930 INFO [RS:1;16b413a53992:33497 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T16:15:00,930 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33497,1731341698716-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,931 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,931 INFO [RS:1;16b413a53992:33497 {}] regionserver.Replication(171): 16b413a53992,33497,1731341698716 started 2024-11-11T16:15:00,946 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,946 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1482): Serving as 16b413a53992,44579,1731341698497, RpcServer on 16b413a53992/172.17.0.2:44579, sessionid=0x1002f9fc70b0001 2024-11-11T16:15:00,947 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T16:15:00,948 DEBUG [RS:0;16b413a53992:44579 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b413a53992,44579,1731341698497 2024-11-11T16:15:00,948 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,44579,1731341698497' 2024-11-11T16:15:00,948 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T16:15:00,949 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T16:15:00,951 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T16:15:00,952 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T16:15:00,952 DEBUG [RS:0;16b413a53992:44579 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b413a53992,44579,1731341698497 2024-11-11T16:15:00,952 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,44579,1731341698497' 2024-11-11T16:15:00,952 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T16:15:00,953 DEBUG [RS:0;16b413a53992:44579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T16:15:00,954 DEBUG [RS:0;16b413a53992:44579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T16:15:00,954 INFO [RS:0;16b413a53992:44579 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T16:15:00,954 INFO [RS:0;16b413a53992:44579 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T16:15:00,955 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,955 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1482): Serving as 16b413a53992,33497,1731341698716, RpcServer on 16b413a53992/172.17.0.2:33497, sessionid=0x1002f9fc70b0002 2024-11-11T16:15:00,955 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T16:15:00,955 DEBUG [RS:1;16b413a53992:33497 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b413a53992,33497,1731341698716 2024-11-11T16:15:00,956 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,33497,1731341698716' 2024-11-11T16:15:00,956 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T16:15:00,956 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T16:15:00,958 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T16:15:00,958 INFO [RS:2;16b413a53992:36245 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T16:15:00,958 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T16:15:00,958 DEBUG [RS:1;16b413a53992:33497 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b413a53992,33497,1731341698716 2024-11-11T16:15:00,958 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,33497,1731341698716' 2024-11-11T16:15:00,958 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,36245,1731341698838-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,958 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T16:15:00,959 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,959 INFO [RS:2;16b413a53992:36245 {}] regionserver.Replication(171): 16b413a53992,36245,1731341698838 started 2024-11-11T16:15:00,966 DEBUG [RS:1;16b413a53992:33497 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T16:15:00,966 DEBUG [RS:1;16b413a53992:33497 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T16:15:00,967 INFO [RS:1;16b413a53992:33497 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T16:15:00,967 INFO [RS:1;16b413a53992:33497 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T16:15:00,989 INFO [RS:2;16b413a53992:36245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:00,990 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1482): Serving as 16b413a53992,36245,1731341698838, RpcServer on 16b413a53992/172.17.0.2:36245, sessionid=0x1002f9fc70b0003 2024-11-11T16:15:00,990 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T16:15:00,990 DEBUG [RS:2;16b413a53992:36245 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b413a53992,36245,1731341698838 2024-11-11T16:15:00,990 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,36245,1731341698838' 2024-11-11T16:15:00,990 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T16:15:00,992 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T16:15:00,993 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T16:15:00,993 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T16:15:00,993 DEBUG [RS:2;16b413a53992:36245 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b413a53992,36245,1731341698838 2024-11-11T16:15:00,993 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,36245,1731341698838' 2024-11-11T16:15:00,993 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T16:15:00,994 DEBUG [RS:2;16b413a53992:36245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T16:15:00,996 DEBUG [RS:2;16b413a53992:36245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T16:15:00,996 INFO [RS:2;16b413a53992:36245 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T16:15:00,996 INFO [RS:2;16b413a53992:36245 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T16:15:01,062 INFO [RS:0;16b413a53992:44579 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:15:01,066 INFO [RS:0;16b413a53992:44579 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C44579%2C1731341698497, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,44579,1731341698497, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs, maxLogs=32 2024-11-11T16:15:01,069 INFO [RS:1;16b413a53992:33497 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:15:01,076 INFO [RS:1;16b413a53992:33497 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C33497%2C1731341698716, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs, maxLogs=32 2024-11-11T16:15:01,099 INFO [RS:2;16b413a53992:36245 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:15:01,111 INFO [RS:2;16b413a53992:36245 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C36245%2C1731341698838, suffix=, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs, maxLogs=32 2024-11-11T16:15:01,127 DEBUG [RS:1;16b413a53992:33497 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716/16b413a53992%2C33497%2C1731341698716.1731341701079, exclude list is [], retry=0 2024-11-11T16:15:01,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:01,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:15:01,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:15:01,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:15:01,155 DEBUG [RS:0;16b413a53992:44579 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,44579,1731341698497/16b413a53992%2C44579%2C1731341698497.1731341701102, exclude list is [], retry=0 2024-11-11T16:15:01,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:15:01,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:15:01,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:15:01,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T16:15:01,176 DEBUG [RS:2;16b413a53992:36245 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838/16b413a53992%2C36245%2C1731341698838.1731341701116, exclude list is [], retry=0 2024-11-11T16:15:01,181 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T16:15:01,181 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T16:15:01,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:15:01,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:15:01,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:15:01,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T16:15:01,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T16:15:01,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T16:15:01,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T16:15:01,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T16:15:01,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T16:15:01,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:01,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:01,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T16:15:01,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T16:15:01,256 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T16:15:01,261 INFO [RS:1;16b413a53992:33497 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716/16b413a53992%2C33497%2C1731341698716.1731341701079 2024-11-11T16:15:01,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T16:15:01,269 INFO [RS:0;16b413a53992:44579 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,44579,1731341698497/16b413a53992%2C44579%2C1731341698497.1731341701102 2024-11-11T16:15:01,276 DEBUG [RS:1;16b413a53992:33497 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33195:33195),(127.0.0.1/127.0.0.1:41647:41647),(127.0.0.1/127.0.0.1:33177:33177)] 2024-11-11T16:15:01,287 INFO [RS:2;16b413a53992:36245 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838/16b413a53992%2C36245%2C1731341698838.1731341701116 2024-11-11T16:15:01,305 DEBUG [RS:0;16b413a53992:44579 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33177:33177),(127.0.0.1/127.0.0.1:41647:41647),(127.0.0.1/127.0.0.1:33195:33195)] 2024-11-11T16:15:01,309 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:01,311 DEBUG [RS:2;16b413a53992:36245 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33177:33177),(127.0.0.1/127.0.0.1:33195:33195),(127.0.0.1/127.0.0.1:41647:41647)] 2024-11-11T16:15:01,314 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70105859, jitterRate=0.04465870559215546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T16:15:01,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731341701140Initializing all the Stores at 1731341701143 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701143Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701165 (+22 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341701165Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701165Cleaning up temporary data from old regions at 1731341701255 (+90 ms)Region opened successfully at 1731341701316 (+61 ms) 2024-11-11T16:15:01,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T16:15:01,329 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T16:15:01,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T16:15:01,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T16:15:01,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T16:15:01,350 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T16:15:01,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731341701329Disabling compacts and flushes for region at 1731341701329Disabling writes for close at 1731341701330 (+1 ms)Writing region close event to WAL at 1731341701349 (+19 ms)Closed at 1731341701350 (+1 ms) 2024-11-11T16:15:01,354 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:01,355 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T16:15:01,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T16:15:01,381 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:01,387 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T16:15:01,540 DEBUG [16b413a53992:35895 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-11T16:15:01,553 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(204): Hosts are {16b413a53992=0} racks are {/default-rack=0} 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T16:15:01,562 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T16:15:01,562 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T16:15:01,562 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T16:15:01,562 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T16:15:01,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:01,580 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,36245,1731341698838, state=OPENING 2024-11-11T16:15:01,585 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T16:15:01,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:01,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:01,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:01,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:01,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:01,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:01,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:01,590 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:01,593 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:01,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:01,782 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T16:15:01,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57361, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T16:15:01,803 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T16:15:01,803 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:15:01,804 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T16:15:01,814 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C36245%2C1731341698838.meta, suffix=.meta, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs, maxLogs=32 2024-11-11T16:15:01,853 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, exclude list is [], retry=0 2024-11-11T16:15:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:15:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:15:01,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:15:01,883 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta 2024-11-11T16:15:01,889 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33177:33177),(127.0.0.1/127.0.0.1:33195:33195),(127.0.0.1/127.0.0.1:41647:41647)] 2024-11-11T16:15:01,889 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:15:01,891 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T16:15:01,895 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T16:15:01,902 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T16:15:01,909 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T16:15:01,909 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:01,910 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T16:15:01,910 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T16:15:01,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T16:15:01,917 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T16:15:01,917 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T16:15:01,922 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T16:15:01,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T16:15:01,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T16:15:01,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T16:15:01,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T16:15:01,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:01,931 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T16:15:01,933 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:01,938 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:01,943 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T16:15:01,944 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T16:15:01,946 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T16:15:01,951 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T16:15:01,954 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59221906, jitterRate=-0.11752483248710632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T16:15:01,955 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T16:15:01,957 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731341701910Writing region info on filesystem at 1731341701911 (+1 ms)Initializing all the Stores at 1731341701914 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701914Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701915 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341701915Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341701915Cleaning up temporary data from old regions at 1731341701944 (+29 ms)Running coprocessor post-open hooks at 1731341701955 (+11 ms)Region opened successfully at 1731341701956 (+1 ms) 2024-11-11T16:15:01,967 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731341701770 2024-11-11T16:15:01,997 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T16:15:01,997 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T16:15:02,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:02,014 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,36245,1731341698838, state=OPEN 2024-11-11T16:15:02,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:02,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:02,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:02,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:02,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16b413a53992,36245,1731341698838 2024-11-11T16:15:02,028 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:02,028 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:02,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:02,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:02,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T16:15:02,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,36245,1731341698838 in 432 msec 2024-11-11T16:15:02,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T16:15:02,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 690 msec 2024-11-11T16:15:02,069 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:02,069 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T16:15:02,115 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:02,123 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1] 2024-11-11T16:15:02,206 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:02,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37967, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:02,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8660 sec 2024-11-11T16:15:02,258 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731341702258, completionTime=-1 2024-11-11T16:15:02,262 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-11T16:15:02,262 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:02,320 INFO [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-11T16:15:02,320 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341762320 2024-11-11T16:15:02,320 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341822320 2024-11-11T16:15:02,320 INFO [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 57 msec 2024-11-11T16:15:02,322 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:02,330 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,331 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,331 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,333 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16b413a53992:35895, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,333 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,336 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,346 DEBUG [master/16b413a53992:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T16:15:02,371 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.359sec 2024-11-11T16:15:02,374 INFO [master/16b413a53992:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T16:15:02,375 INFO [master/16b413a53992:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T16:15:02,376 INFO [master/16b413a53992:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T16:15:02,377 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T16:15:02,377 INFO [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T16:15:02,378 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:15:02,379 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T16:15:02,384 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T16:15:02,385 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T16:15:02,386 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,35895,1731341697364-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:15:02,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b195fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:02,450 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T16:15:02,451 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T16:15:02,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,35895,-1 for getting cluster id 2024-11-11T16:15:02,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:02,470 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '77267fe1-1280-4bf0-aa3a-d386e646f2df' 2024-11-11T16:15:02,473 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "77267fe1-1280-4bf0-aa3a-d386e646f2df" 2024-11-11T16:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e80b5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,35895,-1] 2024-11-11T16:15:02,478 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:02,489 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:02,489 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:02,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a63f137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:02,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1] 2024-11-11T16:15:02,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:02,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:02,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16b413a53992,35895,1731341697364 2024-11-11T16:15:02,601 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:15:02,608 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,35895,1731341697364 2024-11-11T16:15:02,611 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0c4ce2 2024-11-11T16:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:15:02,646 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:15:02,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:15:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:02,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:15:02,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-11T16:15:02,680 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:02,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:15:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:02,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741837_1013 (size=358) 2024-11-11T16:15:02,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741837_1013 (size=358) 2024-11-11T16:15:02,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741837_1013 (size=358) 2024-11-11T16:15:02,784 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62d867b8a89d21a5e2538e1e5222da88, NAME => 'test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,785 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6d05687468bd618dc7fa99d7c6b5f3b2, NAME => 'test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,786 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => aebcd9c83edd713d66ccd5d8f1a5150d, NAME => 'test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:02,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741840_1016 (size=45) 2024-11-11T16:15:02,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741840_1016 (size=45) 2024-11-11T16:15:02,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741840_1016 (size=45) 2024-11-11T16:15:02,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing aebcd9c83edd713d66ccd5d8f1a5150d, disabling compactions & flushes 2024-11-11T16:15:02,845 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:02,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:02,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. after waiting 0 ms 2024-11-11T16:15:02,846 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:02,846 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:02,846 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for aebcd9c83edd713d66ccd5d8f1a5150d: Waiting for close lock at 1731341702845Disabling compacts and flushes for region at 1731341702845Disabling writes for close at 1731341702845Writing region close event to WAL at 1731341702846 (+1 ms)Closed at 1731341702846 2024-11-11T16:15:02,847 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 97c9a93ca402a988eed0542bbe16f17a, NAME => 'test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741838_1014 (size=42) 2024-11-11T16:15:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741838_1014 (size=42) 2024-11-11T16:15:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741838_1014 (size=42) 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 62d867b8a89d21a5e2538e1e5222da88, disabling compactions & flushes 2024-11-11T16:15:02,858 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. after waiting 0 ms 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:02,858 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:02,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62d867b8a89d21a5e2538e1e5222da88: Waiting for close lock at 1731341702858Disabling compacts and flushes for region at 1731341702858Disabling writes for close at 1731341702858Writing region close event to WAL at 1731341702858Closed at 1731341702858 2024-11-11T16:15:02,860 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f2e670caa2864b9154b87162f54c642, NAME => 'test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741839_1015 (size=45) 2024-11-11T16:15:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741839_1015 (size=45) 2024-11-11T16:15:02,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741839_1015 (size=45) 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6d05687468bd618dc7fa99d7c6b5f3b2, disabling compactions & flushes 2024-11-11T16:15:02,899 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. after waiting 0 ms 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:02,899 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:02,899 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6d05687468bd618dc7fa99d7c6b5f3b2: Waiting for close lock at 1731341702899Disabling compacts and flushes for region at 1731341702899Disabling writes for close at 1731341702899Writing region close event to WAL at 1731341702899Closed at 1731341702899 2024-11-11T16:15:02,900 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b7e69cf109199b3c3d754201ea9818cd, NAME => 'test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741842_1018 (size=45) 2024-11-11T16:15:02,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741842_1018 (size=45) 2024-11-11T16:15:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741842_1018 (size=45) 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 97c9a93ca402a988eed0542bbe16f17a, disabling compactions & flushes 2024-11-11T16:15:02,904 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. after waiting 0 ms 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:02,904 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:02,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 97c9a93ca402a988eed0542bbe16f17a: Waiting for close lock at 1731341702904Disabling compacts and flushes for region at 1731341702904Disabling writes for close at 1731341702904Writing region close event to WAL at 1731341702904Closed at 1731341702904 2024-11-11T16:15:02,905 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 756a2cfebbc98a6396c782a802598ffc, NAME => 'test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741841_1017 (size=45) 2024-11-11T16:15:02,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741841_1017 (size=45) 2024-11-11T16:15:02,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741841_1017 (size=45) 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5f2e670caa2864b9154b87162f54c642, disabling compactions & flushes 2024-11-11T16:15:02,927 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. after waiting 0 ms 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:02,927 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:02,927 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f2e670caa2864b9154b87162f54c642: Waiting for close lock at 1731341702927Disabling compacts and flushes for region at 1731341702927Disabling writes for close at 1731341702927Writing region close event to WAL at 1731341702927Closed at 1731341702927 2024-11-11T16:15:02,928 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8845a47073d69ee0a0021d18e08e51ef, NAME => 'test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741844_1020 (size=45) 2024-11-11T16:15:02,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741844_1020 (size=45) 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 756a2cfebbc98a6396c782a802598ffc, disabling compactions & flushes 2024-11-11T16:15:02,945 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. after waiting 0 ms 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:02,945 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:02,945 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 756a2cfebbc98a6396c782a802598ffc: Waiting for close lock at 1731341702945Disabling compacts and flushes for region at 1731341702945Disabling writes for close at 1731341702945Writing region close event to WAL at 1731341702945Closed at 1731341702945 2024-11-11T16:15:02,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741844_1020 (size=45) 2024-11-11T16:15:02,946 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cd1d8bee83938f0837b5cdacde7e89c4, NAME => 'test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741843_1019 (size=45) 2024-11-11T16:15:02,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741843_1019 (size=45) 2024-11-11T16:15:02,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741843_1019 (size=45) 2024-11-11T16:15:02,977 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,977 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b7e69cf109199b3c3d754201ea9818cd, disabling compactions & flushes 2024-11-11T16:15:02,977 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:02,977 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:02,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. after waiting 0 ms 2024-11-11T16:15:02,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:02,978 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:02,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b7e69cf109199b3c3d754201ea9818cd: Waiting for close lock at 1731341702977Disabling compacts and flushes for region at 1731341702977Disabling writes for close at 1731341702978 (+1 ms)Writing region close event to WAL at 1731341702978Closed at 1731341702978 2024-11-11T16:15:02,978 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5317ee89294e310637e2785c5a255b6b, NAME => 'test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741845_1021 (size=45) 2024-11-11T16:15:02,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741845_1021 (size=45) 2024-11-11T16:15:02,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741845_1021 (size=45) 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8845a47073d69ee0a0021d18e08e51ef, disabling compactions & flushes 2024-11-11T16:15:02,991 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. after waiting 0 ms 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:02,991 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:02,991 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8845a47073d69ee0a0021d18e08e51ef: Waiting for close lock at 1731341702991Disabling compacts and flushes for region at 1731341702991Disabling writes for close at 1731341702991Writing region close event to WAL at 1731341702991Closed at 1731341702991 2024-11-11T16:15:02,992 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ba7b9cc976155fbb76bf00c2ffd4495, NAME => 'test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:02,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741846_1022 (size=45) 2024-11-11T16:15:02,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741846_1022 (size=45) 2024-11-11T16:15:03,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741846_1022 (size=45) 2024-11-11T16:15:03,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cd1d8bee83938f0837b5cdacde7e89c4, disabling compactions & flushes 2024-11-11T16:15:03,001 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:03,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:03,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. after waiting 0 ms 2024-11-11T16:15:03,002 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:03,002 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:03,002 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cd1d8bee83938f0837b5cdacde7e89c4: Waiting for close lock at 1731341703001Disabling compacts and flushes for region at 1731341703001Disabling writes for close at 1731341703001Writing region close event to WAL at 1731341703002 (+1 ms)Closed at 1731341703002 2024-11-11T16:15:03,003 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4f42aec7889fcb07b96681cea332b42d, NAME => 'test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741847_1023 (size=45) 2024-11-11T16:15:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741847_1023 (size=45) 2024-11-11T16:15:03,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741847_1023 (size=45) 2024-11-11T16:15:03,013 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5317ee89294e310637e2785c5a255b6b, disabling compactions & flushes 2024-11-11T16:15:03,014 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:03,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:03,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. after waiting 0 ms 2024-11-11T16:15:03,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:03,014 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:03,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5317ee89294e310637e2785c5a255b6b: Waiting for close lock at 1731341703014Disabling compacts and flushes for region at 1731341703014Disabling writes for close at 1731341703014Writing region close event to WAL at 1731341703014Closed at 1731341703014 2024-11-11T16:15:03,015 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 58c93273fb8e84a5a8b730fc9fafb6f7, NAME => 'test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:03,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741848_1024 (size=45) 2024-11-11T16:15:03,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741848_1024 (size=45) 2024-11-11T16:15:03,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741848_1024 (size=45) 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7ba7b9cc976155fbb76bf00c2ffd4495, disabling compactions & flushes 2024-11-11T16:15:03,042 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. after waiting 0 ms 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:03,042 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:03,042 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ba7b9cc976155fbb76bf00c2ffd4495: Waiting for close lock at 1731341703042Disabling compacts and flushes for region at 1731341703042Disabling writes for close at 1731341703042Writing region close event to WAL at 1731341703042Closed at 1731341703042 2024-11-11T16:15:03,043 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7df81ce6daca1d892d80d840de0c0b8b, NAME => 'test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741849_1025 (size=45) 2024-11-11T16:15:03,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741849_1025 (size=45) 2024-11-11T16:15:03,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741849_1025 (size=45) 2024-11-11T16:15:03,065 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4f42aec7889fcb07b96681cea332b42d, disabling compactions & flushes 2024-11-11T16:15:03,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:03,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:03,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. after waiting 0 ms 2024-11-11T16:15:03,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:03,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:03,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4f42aec7889fcb07b96681cea332b42d: Waiting for close lock at 1731341703066Disabling compacts and flushes for region at 1731341703066Disabling writes for close at 1731341703066Writing region close event to WAL at 1731341703066Closed at 1731341703066 2024-11-11T16:15:03,067 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 98ffcdd491bf651a96a2bf2c2b5a9a95, NAME => 'test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741850_1026 (size=45) 2024-11-11T16:15:03,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741850_1026 (size=45) 2024-11-11T16:15:03,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741850_1026 (size=45) 2024-11-11T16:15:03,085 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,086 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 58c93273fb8e84a5a8b730fc9fafb6f7, disabling compactions & flushes 2024-11-11T16:15:03,086 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:03,086 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:03,086 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. after waiting 0 ms 2024-11-11T16:15:03,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:03,088 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:03,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 58c93273fb8e84a5a8b730fc9fafb6f7: Waiting for close lock at 1731341703086Disabling compacts and flushes for region at 1731341703086Disabling writes for close at 1731341703086Writing region close event to WAL at 1731341703088 (+2 ms)Closed at 1731341703088 2024-11-11T16:15:03,089 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f9f064481e2ac446aeb716014e0cdf6b, NAME => 'test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741851_1027 (size=45) 2024-11-11T16:15:03,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741851_1027 (size=45) 2024-11-11T16:15:03,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741851_1027 (size=45) 2024-11-11T16:15:03,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7df81ce6daca1d892d80d840de0c0b8b, disabling compactions & flushes 2024-11-11T16:15:03,104 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:03,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:03,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. after waiting 0 ms 2024-11-11T16:15:03,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:03,105 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:03,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7df81ce6daca1d892d80d840de0c0b8b: Waiting for close lock at 1731341703104Disabling compacts and flushes for region at 1731341703104Disabling writes for close at 1731341703105 (+1 ms)Writing region close event to WAL at 1731341703105Closed at 1731341703105 2024-11-11T16:15:03,106 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 35b42287bbbc0c5919124a8c4b800ea1, NAME => 'test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741852_1028 (size=45) 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 98ffcdd491bf651a96a2bf2c2b5a9a95, disabling compactions & flushes 2024-11-11T16:15:03,123 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:03,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741852_1028 (size=45) 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. after waiting 0 ms 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:03,123 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:03,123 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 98ffcdd491bf651a96a2bf2c2b5a9a95: Waiting for close lock at 1731341703123Disabling compacts and flushes for region at 1731341703123Disabling writes for close at 1731341703123Writing region close event to WAL at 1731341703123Closed at 1731341703123 2024-11-11T16:15:03,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741852_1028 (size=45) 2024-11-11T16:15:03,124 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e3cef8c6c3398e017e8dfa5cd0c9c242, NAME => 'test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741853_1029 (size=45) 2024-11-11T16:15:03,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741853_1029 (size=45) 2024-11-11T16:15:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741853_1029 (size=45) 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f9f064481e2ac446aeb716014e0cdf6b, disabling compactions & flushes 2024-11-11T16:15:03,156 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. after waiting 0 ms 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:03,156 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:03,156 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f9f064481e2ac446aeb716014e0cdf6b: Waiting for close lock at 1731341703156Disabling compacts and flushes for region at 1731341703156Disabling writes for close at 1731341703156Writing region close event to WAL at 1731341703156Closed at 1731341703156 2024-11-11T16:15:03,157 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d939495820cb0d73d77301eb7bbcb7fb, NAME => 'test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741855_1031 (size=45) 2024-11-11T16:15:03,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741855_1031 (size=45) 2024-11-11T16:15:03,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e3cef8c6c3398e017e8dfa5cd0c9c242, disabling compactions & flushes 2024-11-11T16:15:03,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:03,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:03,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741855_1031 (size=45) 2024-11-11T16:15:03,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. after waiting 0 ms 2024-11-11T16:15:03,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:03,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:03,195 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e3cef8c6c3398e017e8dfa5cd0c9c242: Waiting for close lock at 1731341703194Disabling compacts and flushes for region at 1731341703194Disabling writes for close at 1731341703194Writing region close event to WAL at 1731341703194Closed at 1731341703194 2024-11-11T16:15:03,195 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 60aabcb1be5df802409ef5ef8a1eccba, NAME => 'test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741854_1030 (size=45) 2024-11-11T16:15:03,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741854_1030 (size=45) 2024-11-11T16:15:03,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741854_1030 (size=45) 2024-11-11T16:15:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741856_1032 (size=45) 2024-11-11T16:15:03,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741856_1032 (size=45) 2024-11-11T16:15:03,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741856_1032 (size=45) 2024-11-11T16:15:03,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d939495820cb0d73d77301eb7bbcb7fb, disabling compactions & flushes 2024-11-11T16:15:03,230 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:03,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:03,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. after waiting 0 ms 2024-11-11T16:15:03,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:03,231 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:03,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d939495820cb0d73d77301eb7bbcb7fb: Waiting for close lock at 1731341703230Disabling compacts and flushes for region at 1731341703230Disabling writes for close at 1731341703230Writing region close event to WAL at 1731341703231 (+1 ms)Closed at 1731341703231 2024-11-11T16:15:03,232 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e762741692724ea54cb4ce74a7e615ae, NAME => 'test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741857_1033 (size=45) 2024-11-11T16:15:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741857_1033 (size=45) 2024-11-11T16:15:03,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741857_1033 (size=45) 2024-11-11T16:15:03,247 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,247 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 60aabcb1be5df802409ef5ef8a1eccba, disabling compactions & flushes 2024-11-11T16:15:03,247 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:03,247 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:03,247 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. after waiting 0 ms 2024-11-11T16:15:03,248 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:03,248 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:03,248 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 60aabcb1be5df802409ef5ef8a1eccba: Waiting for close lock at 1731341703247Disabling compacts and flushes for region at 1731341703247Disabling writes for close at 1731341703247Writing region close event to WAL at 1731341703248 (+1 ms)Closed at 1731341703248 2024-11-11T16:15:03,249 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4c2941f7d018cc329ed386329a479fbf, NAME => 'test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741859_1035 (size=45) 2024-11-11T16:15:03,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741859_1035 (size=45) 2024-11-11T16:15:03,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4c2941f7d018cc329ed386329a479fbf, disabling compactions & flushes 2024-11-11T16:15:03,279 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:03,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:03,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. after waiting 0 ms 2024-11-11T16:15:03,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:03,279 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:03,280 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4c2941f7d018cc329ed386329a479fbf: Waiting for close lock at 1731341703279Disabling compacts and flushes for region at 1731341703279Disabling writes for close at 1731341703279Writing region close event to WAL at 1731341703279Closed at 1731341703279 2024-11-11T16:15:03,280 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9984848611dae4de6b309aeff78ed621, NAME => 'test,www,1731341702648.9984848611dae4de6b309aeff78ed621.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741859_1035 (size=45) 2024-11-11T16:15:03,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741858_1034 (size=45) 2024-11-11T16:15:03,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741858_1034 (size=45) 2024-11-11T16:15:03,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741858_1034 (size=45) 2024-11-11T16:15:03,290 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,290 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e762741692724ea54cb4ce74a7e615ae, disabling compactions & flushes 2024-11-11T16:15:03,290 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:03,290 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:03,291 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. after waiting 0 ms 2024-11-11T16:15:03,291 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:03,291 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:03,291 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e762741692724ea54cb4ce74a7e615ae: Waiting for close lock at 1731341703290Disabling compacts and flushes for region at 1731341703290Disabling writes for close at 1731341703291 (+1 ms)Writing region close event to WAL at 1731341703291Closed at 1731341703291 2024-11-11T16:15:03,291 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7fb28854349ec0d596f8f0dca65b13b8, NAME => 'test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741860_1036 (size=45) 2024-11-11T16:15:03,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741860_1036 (size=45) 2024-11-11T16:15:03,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741861_1037 (size=45) 2024-11-11T16:15:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731341702648.9984848611dae4de6b309aeff78ed621.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9984848611dae4de6b309aeff78ed621, disabling compactions & flushes 2024-11-11T16:15:03,336 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731341702648.9984848611dae4de6b309aeff78ed621. after waiting 0 ms 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:03,336 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:03,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9984848611dae4de6b309aeff78ed621: Waiting for close lock at 1731341703336Disabling compacts and flushes for region at 1731341703336Disabling writes for close at 1731341703336Writing region close event to WAL at 1731341703336Closed at 1731341703336 2024-11-11T16:15:03,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741860_1036 (size=45) 2024-11-11T16:15:03,337 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e62b8abd36e22d5d589c5bdd191ed105, NAME => 'test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741861_1037 (size=45) 2024-11-11T16:15:03,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741861_1037 (size=45) 2024-11-11T16:15:03,344 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7fb28854349ec0d596f8f0dca65b13b8, disabling compactions & flushes 2024-11-11T16:15:03,345 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:03,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:03,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. after waiting 0 ms 2024-11-11T16:15:03,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:03,345 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:03,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7fb28854349ec0d596f8f0dca65b13b8: Waiting for close lock at 1731341703345Disabling compacts and flushes for region at 1731341703345Disabling writes for close at 1731341703345Writing region close event to WAL at 1731341703345Closed at 1731341703345 2024-11-11T16:15:03,346 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7ee84052923dbaf7f5c9581236276c4c, NAME => 'test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba 2024-11-11T16:15:03,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741862_1038 (size=45) 2024-11-11T16:15:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741862_1038 (size=45) 2024-11-11T16:15:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741862_1038 (size=45) 2024-11-11T16:15:03,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e62b8abd36e22d5d589c5bdd191ed105, disabling compactions & flushes 2024-11-11T16:15:03,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:03,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:03,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. after waiting 0 ms 2024-11-11T16:15:03,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:03,412 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:03,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e62b8abd36e22d5d589c5bdd191ed105: Waiting for close lock at 1731341703411Disabling compacts and flushes for region at 1731341703411Disabling writes for close at 1731341703412 (+1 ms)Writing region close event to WAL at 1731341703412Closed at 1731341703412 2024-11-11T16:15:03,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741863_1039 (size=42) 2024-11-11T16:15:03,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741863_1039 (size=42) 2024-11-11T16:15:03,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741863_1039 (size=42) 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7ee84052923dbaf7f5c9581236276c4c, disabling compactions & flushes 2024-11-11T16:15:03,428 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. after waiting 0 ms 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:03,428 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:03,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7ee84052923dbaf7f5c9581236276c4c: Waiting for close lock at 1731341703428Disabling compacts and flushes for region at 1731341703428Disabling writes for close at 1731341703428Writing region close event to WAL at 1731341703428Closed at 1731341703428 2024-11-11T16:15:03,599 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:03,599 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 35b42287bbbc0c5919124a8c4b800ea1, disabling compactions & flushes 2024-11-11T16:15:03,599 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:03,599 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:03,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. after waiting 1 ms 2024-11-11T16:15:03,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:03,600 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:03,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 35b42287bbbc0c5919124a8c4b800ea1: Waiting for close lock at 1731341703599Disabling compacts and flushes for region at 1731341703599Disabling writes for close at 1731341703600 (+1 ms)Writing region close event to WAL at 1731341703600Closed at 1731341703600 2024-11-11T16:15:03,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:15:03,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731341702648.9984848611dae4de6b309aeff78ed621.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731341703603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341703603"}]},"ts":"1731341703603"} 2024-11-11T16:15:03,688 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-11T16:15:03,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:15:03,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341703692"}]},"ts":"1731341703692"} 2024-11-11T16:15:03,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-11T16:15:03,708 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {16b413a53992=0} racks are {/default-rack=0} 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T16:15:03,743 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T16:15:03,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T16:15:03,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T16:15:03,744 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T16:15:03,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T16:15:03,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62d867b8a89d21a5e2538e1e5222da88, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d05687468bd618dc7fa99d7c6b5f3b2, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f2e670caa2864b9154b87162f54c642, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=756a2cfebbc98a6396c782a802598ffc, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8845a47073d69ee0a0021d18e08e51ef, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd1d8bee83938f0837b5cdacde7e89c4, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ba7b9cc976155fbb76bf00c2ffd4495, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f42aec7889fcb07b96681cea332b42d, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7df81ce6daca1d892d80d840de0c0b8b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=98ffcdd491bf651a96a2bf2c2b5a9a95, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=35b42287bbbc0c5919124a8c4b800ea1, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e3cef8c6c3398e017e8dfa5cd0c9c242, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d939495820cb0d73d77301eb7bbcb7fb, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e762741692724ea54cb4ce74a7e615ae, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c2941f7d018cc329ed386329a479fbf, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7fb28854349ec0d596f8f0dca65b13b8, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e62b8abd36e22d5d589c5bdd191ed105, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN}] 2024-11-11T16:15:03,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:03,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:03,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e62b8abd36e22d5d589c5bdd191ed105, ASSIGN 2024-11-11T16:15:03,805 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e62b8abd36e22d5d589c5bdd191ed105, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:03,809 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7fb28854349ec0d596f8f0dca65b13b8, ASSIGN 2024-11-11T16:15:03,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7fb28854349ec0d596f8f0dca65b13b8, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:03,818 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:03,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:03,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c2941f7d018cc329ed386329a479fbf, ASSIGN 2024-11-11T16:15:03,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c2941f7d018cc329ed386329a479fbf, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:03,846 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e762741692724ea54cb4ce74a7e615ae, ASSIGN 2024-11-11T16:15:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:03,851 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e762741692724ea54cb4ce74a7e615ae, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:03,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, ASSIGN 2024-11-11T16:15:03,859 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:03,868 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d939495820cb0d73d77301eb7bbcb7fb, ASSIGN 2024-11-11T16:15:03,873 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d939495820cb0d73d77301eb7bbcb7fb, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:03,879 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e3cef8c6c3398e017e8dfa5cd0c9c242, ASSIGN 2024-11-11T16:15:03,887 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e3cef8c6c3398e017e8dfa5cd0c9c242, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:03,908 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=35b42287bbbc0c5919124a8c4b800ea1, ASSIGN 2024-11-11T16:15:03,911 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=35b42287bbbc0c5919124a8c4b800ea1, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:03,918 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:03,925 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:03,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=98ffcdd491bf651a96a2bf2c2b5a9a95, ASSIGN 2024-11-11T16:15:03,935 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=98ffcdd491bf651a96a2bf2c2b5a9a95, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:03,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7df81ce6daca1d892d80d840de0c0b8b, ASSIGN 2024-11-11T16:15:03,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7df81ce6daca1d892d80d840de0c0b8b, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:03,951 INFO [16b413a53992:35895 {}] balancer.BaseLoadBalancer(388): Reassigned 13 regions. 13 retained the pre-restart assignment. 2024-11-11T16:15:03,954 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:03,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4c2941f7d018cc329ed386329a479fbf, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:03,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:03,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7fb28854349ec0d596f8f0dca65b13b8, ASSIGN because future has completed 2024-11-11T16:15:03,972 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:03,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c2941f7d018cc329ed386329a479fbf, ASSIGN because future has completed 2024-11-11T16:15:03,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb28854349ec0d596f8f0dca65b13b8, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:03,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e3cef8c6c3398e017e8dfa5cd0c9c242, ASSIGN because future has completed 2024-11-11T16:15:04,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=35b42287bbbc0c5919124a8c4b800ea1, ASSIGN because future has completed 2024-11-11T16:15:04,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35b42287bbbc0c5919124a8c4b800ea1, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3cef8c6c3398e017e8dfa5cd0c9c242, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c2941f7d018cc329ed386329a479fbf, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7df81ce6daca1d892d80d840de0c0b8b, ASSIGN because future has completed 2024-11-11T16:15:04,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7df81ce6daca1d892d80d840de0c0b8b, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN because future has completed 2024-11-11T16:15:04,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9984848611dae4de6b309aeff78ed621, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e762741692724ea54cb4ce74a7e615ae, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:04,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN because future has completed 2024-11-11T16:15:04,101 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:04,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, ASSIGN because future has completed 2024-11-11T16:15:04,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN because future has completed 2024-11-11T16:15:04,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e62b8abd36e22d5d589c5bdd191ed105, ASSIGN because future has completed 2024-11-11T16:15:04,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e762741692724ea54cb4ce74a7e615ae, ASSIGN because future has completed 2024-11-11T16:15:04,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d939495820cb0d73d77301eb7bbcb7fb, ASSIGN because future has completed 2024-11-11T16:15:04,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=98ffcdd491bf651a96a2bf2c2b5a9a95, ASSIGN because future has completed 2024-11-11T16:15:04,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98ffcdd491bf651a96a2bf2c2b5a9a95, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d939495820cb0d73d77301eb7bbcb7fb, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e762741692724ea54cb4ce74a7e615ae, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e62b8abd36e22d5d589c5bdd191ed105, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,168 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T16:15:04,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49561, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T16:15:04,184 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f42aec7889fcb07b96681cea332b42d, ASSIGN 2024-11-11T16:15:04,186 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f42aec7889fcb07b96681cea332b42d, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:04,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ba7b9cc976155fbb76bf00c2ffd4495, ASSIGN 2024-11-11T16:15:04,191 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ba7b9cc976155fbb76bf00c2ffd4495, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:04,192 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:04,192 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 35b42287bbbc0c5919124a8c4b800ea1, NAME => 'test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-11T16:15:04,193 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,193 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:04,194 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,194 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,195 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:04,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd1d8bee83938f0837b5cdacde7e89c4, ASSIGN 2024-11-11T16:15:04,201 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd1d8bee83938f0837b5cdacde7e89c4, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:04,203 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8845a47073d69ee0a0021d18e08e51ef, ASSIGN 2024-11-11T16:15:04,205 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8845a47073d69ee0a0021d18e08e51ef, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:04,205 INFO [StoreOpener-35b42287bbbc0c5919124a8c4b800ea1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,209 INFO [StoreOpener-35b42287bbbc0c5919124a8c4b800ea1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35b42287bbbc0c5919124a8c4b800ea1 columnFamilyName family 2024-11-11T16:15:04,209 DEBUG [StoreOpener-35b42287bbbc0c5919124a8c4b800ea1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,211 INFO [StoreOpener-35b42287bbbc0c5919124a8c4b800ea1-1 {}] regionserver.HStore(327): Store=35b42287bbbc0c5919124a8c4b800ea1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,211 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,212 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,213 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=756a2cfebbc98a6396c782a802598ffc, ASSIGN 2024-11-11T16:15:04,213 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,215 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,216 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,218 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=756a2cfebbc98a6396c782a802598ffc, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:04,221 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:04,223 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:04,225 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f2e670caa2864b9154b87162f54c642, ASSIGN 2024-11-11T16:15:04,227 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5f2e670caa2864b9154b87162f54c642, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:04,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:04,230 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/35b42287bbbc0c5919124a8c4b800ea1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,230 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:04,231 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 35b42287bbbc0c5919124a8c4b800ea1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59934135, jitterRate=-0.1069117933511734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,231 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:04,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:04,232 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 35b42287bbbc0c5919124a8c4b800ea1: Running coprocessor pre-open hook at 1731341704194Writing region info on filesystem at 1731341704194Initializing all the Stores at 1731341704203 (+9 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704203Cleaning up temporary data from old regions at 1731341704216 (+13 ms)Running coprocessor post-open hooks at 1731341704231 (+15 ms)Region opened successfully at 1731341704232 (+1 ms) 2024-11-11T16:15:04,235 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1., pid=32, masterSystemTime=1731341704168 2024-11-11T16:15:04,235 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN; state=OFFLINE, location=16b413a53992,36245,1731341698838; forceNewPlan=false, retain=false 2024-11-11T16:15:04,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d05687468bd618dc7fa99d7c6b5f3b2, ASSIGN 2024-11-11T16:15:04,239 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:04,239 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:04,239 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:04,240 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => e3cef8c6c3398e017e8dfa5cd0c9c242, NAME => 'test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-11T16:15:04,240 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d05687468bd618dc7fa99d7c6b5f3b2, ASSIGN; state=OFFLINE, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:04,240 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,240 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,241 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,241 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,252 INFO [16b413a53992:35895 {}] balancer.BaseLoadBalancer(388): Reassigned 12 regions. 12 retained the pre-restart assignment. 2024-11-11T16:15:04,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35b42287bbbc0c5919124a8c4b800ea1, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:04,254 INFO [StoreOpener-e3cef8c6c3398e017e8dfa5cd0c9c242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,258 INFO [StoreOpener-e3cef8c6c3398e017e8dfa5cd0c9c242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3cef8c6c3398e017e8dfa5cd0c9c242 columnFamilyName family 2024-11-11T16:15:04,258 DEBUG [StoreOpener-e3cef8c6c3398e017e8dfa5cd0c9c242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,260 INFO [StoreOpener-e3cef8c6c3398e017e8dfa5cd0c9c242-1 {}] regionserver.HStore(327): Store=e3cef8c6c3398e017e8dfa5cd0c9c242/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,261 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,263 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,264 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,265 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,265 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,268 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,273 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e3cef8c6c3398e017e8dfa5cd0c9c242/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,274 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened e3cef8c6c3398e017e8dfa5cd0c9c242; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73469268, jitterRate=0.09477740526199341}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,274 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:04,274 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for e3cef8c6c3398e017e8dfa5cd0c9c242: Running coprocessor pre-open hook at 1731341704242Writing region info on filesystem at 1731341704242Initializing all the Stores at 1731341704244 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704244Cleaning up temporary data from old regions at 1731341704265 (+21 ms)Running coprocessor post-open hooks at 1731341704274 (+9 ms)Region opened successfully at 1731341704274 2024-11-11T16:15:04,276 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242., pid=33, masterSystemTime=1731341704168 2024-11-11T16:15:04,281 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:04,282 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 60aabcb1be5df802409ef5ef8a1eccba, NAME => 'test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-11T16:15:04,283 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T16:15:04,283 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,283 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,283 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,283 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,286 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:04,287 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:04,287 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:04,287 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c2941f7d018cc329ed386329a479fbf, NAME => 'test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-11T16:15:04,287 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,288 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,288 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,288 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-11T16:15:04,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 35b42287bbbc0c5919124a8c4b800ea1, server=16b413a53992,44579,1731341698497 in 253 msec 2024-11-11T16:15:04,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47355, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T16:15:04,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=35b42287bbbc0c5919124a8c4b800ea1, ASSIGN in 541 msec 2024-11-11T16:15:04,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,305 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,308 INFO [StoreOpener-4c2941f7d018cc329ed386329a479fbf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=8845a47073d69ee0a0021d18e08e51ef, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,312 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60aabcb1be5df802409ef5ef8a1eccba columnFamilyName family 2024-11-11T16:15:04,312 DEBUG [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5f2e670caa2864b9154b87162f54c642, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,319 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] regionserver.HStore(327): Store=60aabcb1be5df802409ef5ef8a1eccba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,319 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e3cef8c6c3398e017e8dfa5cd0c9c242, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:04,321 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,321 INFO [StoreOpener-4c2941f7d018cc329ed386329a479fbf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c2941f7d018cc329ed386329a479fbf columnFamilyName family 2024-11-11T16:15:04,322 DEBUG [StoreOpener-4c2941f7d018cc329ed386329a479fbf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,322 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:04,322 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,322 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => d939495820cb0d73d77301eb7bbcb7fb, NAME => 'test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-11T16:15:04,322 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,322 INFO [StoreOpener-4c2941f7d018cc329ed386329a479fbf-1 {}] regionserver.HStore(327): Store=4c2941f7d018cc329ed386329a479fbf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,322 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,323 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,323 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,323 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,323 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,323 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,324 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,326 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,326 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,327 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,327 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ba7b9cc976155fbb76bf00c2ffd4495, ASSIGN because future has completed 2024-11-11T16:15:04,332 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,332 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,334 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 60aabcb1be5df802409ef5ef8a1eccba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59704828, jitterRate=-0.11032873392105103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,335 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:04,335 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 60aabcb1be5df802409ef5ef8a1eccba: Running coprocessor pre-open hook at 1731341704283Writing region info on filesystem at 1731341704283Initializing all the Stores at 1731341704286 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704286Cleaning up temporary data from old regions at 1731341704323 (+37 ms)Running coprocessor post-open hooks at 1731341704335 (+12 ms)Region opened successfully at 1731341704335 2024-11-11T16:15:04,336 INFO [StoreOpener-d939495820cb0d73d77301eb7bbcb7fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8845a47073d69ee0a0021d18e08e51ef, ASSIGN because future has completed 2024-11-11T16:15:04,341 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba., pid=37, masterSystemTime=1731341704252 2024-11-11T16:15:04,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5f2e670caa2864b9154b87162f54c642, ASSIGN because future has completed 2024-11-11T16:15:04,341 INFO [StoreOpener-d939495820cb0d73d77301eb7bbcb7fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d939495820cb0d73d77301eb7bbcb7fb columnFamilyName family 2024-11-11T16:15:04,342 DEBUG [StoreOpener-d939495820cb0d73d77301eb7bbcb7fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,346 INFO [StoreOpener-d939495820cb0d73d77301eb7bbcb7fb-1 {}] regionserver.HStore(327): Store=d939495820cb0d73d77301eb7bbcb7fb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN because future has completed 2024-11-11T16:15:04,347 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:04,347 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:04,347 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:04,348 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ee84052923dbaf7f5c9581236276c4c, NAME => 'test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-11T16:15:04,348 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,348 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,348 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN because future has completed 2024-11-11T16:15:04,349 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,349 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,350 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=22 2024-11-11T16:15:04,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure e3cef8c6c3398e017e8dfa5cd0c9c242, server=16b413a53992,44579,1731341698497 in 303 msec 2024-11-11T16:15:04,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN because future has completed 2024-11-11T16:15:04,357 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN because future has completed 2024-11-11T16:15:04,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e3cef8c6c3398e017e8dfa5cd0c9c242, ASSIGN in 600 msec 2024-11-11T16:15:04,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,360 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN because future has completed 2024-11-11T16:15:04,360 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,365 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4c2941f7d018cc329ed386329a479fbf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,366 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 4c2941f7d018cc329ed386329a479fbf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74766278, jitterRate=0.11410436034202576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,367 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:04,367 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,367 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 4c2941f7d018cc329ed386329a479fbf: Running coprocessor pre-open hook at 1731341704288Writing region info on filesystem at 1731341704288Initializing all the Stores at 1731341704290 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704290Cleaning up temporary data from old regions at 1731341704327 (+37 ms)Running coprocessor post-open hooks at 1731341704367 (+40 ms)Region opened successfully at 1731341704367 2024-11-11T16:15:04,368 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf., pid=34, masterSystemTime=1731341704168 2024-11-11T16:15:04,372 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:04,372 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:04,372 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:04,373 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fb28854349ec0d596f8f0dca65b13b8, NAME => 'test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-11T16:15:04,373 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,373 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,373 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,373 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,377 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,378 INFO [StoreOpener-7fb28854349ec0d596f8f0dca65b13b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,379 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,44579,1731341698497, table=test, region=4c2941f7d018cc329ed386329a479fbf. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,380 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/d939495820cb0d73d77301eb7bbcb7fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,381 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened d939495820cb0d73d77301eb7bbcb7fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62072793, jitterRate=-0.07504330575466156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,381 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:04,381 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for d939495820cb0d73d77301eb7bbcb7fb: Running coprocessor pre-open hook at 1731341704323Writing region info on filesystem at 1731341704323Initializing all the Stores at 1731341704326 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704326Cleaning up temporary data from old regions at 1731341704361 (+35 ms)Running coprocessor post-open hooks at 1731341704381 (+20 ms)Region opened successfully at 1731341704381 2024-11-11T16:15:04,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c2941f7d018cc329ed386329a479fbf, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:04,383 INFO [StoreOpener-7fb28854349ec0d596f8f0dca65b13b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fb28854349ec0d596f8f0dca65b13b8 columnFamilyName family 2024-11-11T16:15:04,383 DEBUG [StoreOpener-7fb28854349ec0d596f8f0dca65b13b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,384 INFO [StoreOpener-7fb28854349ec0d596f8f0dca65b13b8-1 {}] regionserver.HStore(327): Store=7fb28854349ec0d596f8f0dca65b13b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,384 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ee84052923dbaf7f5c9581236276c4c columnFamilyName family 2024-11-11T16:15:04,384 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,384 DEBUG [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,384 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb., pid=39, masterSystemTime=1731341704282 2024-11-11T16:15:04,385 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] regionserver.HStore(327): Store=7ee84052923dbaf7f5c9581236276c4c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,386 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,387 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,387 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,388 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,388 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:04,388 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:04,389 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => e762741692724ea54cb4ce74a7e615ae, NAME => 'test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,390 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,390 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,393 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,394 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,394 INFO [StoreOpener-e762741692724ea54cb4ce74a7e615ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=26 2024-11-11T16:15:04,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c2941f7d018cc329ed386329a479fbf, server=16b413a53992,44579,1731341698497 in 355 msec 2024-11-11T16:15:04,401 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7fb28854349ec0d596f8f0dca65b13b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,402 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,402 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 7fb28854349ec0d596f8f0dca65b13b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67266604, jitterRate=0.0023505091667175293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,403 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:04,403 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 7fb28854349ec0d596f8f0dca65b13b8: Running coprocessor pre-open hook at 1731341704374Writing region info on filesystem at 1731341704374Initializing all the Stores at 1731341704376 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704376Cleaning up temporary data from old regions at 1731341704390 (+14 ms)Running coprocessor post-open hooks at 1731341704403 (+13 ms)Region opened successfully at 1731341704403 2024-11-11T16:15:04,404 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 7ee84052923dbaf7f5c9581236276c4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63878977, jitterRate=-0.048129066824913025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4c2941f7d018cc329ed386329a479fbf, ASSIGN in 648 msec 2024-11-11T16:15:04,404 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:04,404 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 7ee84052923dbaf7f5c9581236276c4c: Running coprocessor pre-open hook at 1731341704349Writing region info on filesystem at 1731341704349Initializing all the Stores at 1731341704356 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704356Cleaning up temporary data from old regions at 1731341704389 (+33 ms)Running coprocessor post-open hooks at 1731341704404 (+15 ms)Region opened successfully at 1731341704404 2024-11-11T16:15:04,405 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8., pid=31, masterSystemTime=1731341704168 2024-11-11T16:15:04,406 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c., pid=36, masterSystemTime=1731341704252 2024-11-11T16:15:04,407 INFO [StoreOpener-e762741692724ea54cb4ce74a7e615ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e762741692724ea54cb4ce74a7e615ae columnFamilyName family 2024-11-11T16:15:04,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,408 DEBUG [StoreOpener-e762741692724ea54cb4ce74a7e615ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,409 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:04,409 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:04,410 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:04,411 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 7df81ce6daca1d892d80d840de0c0b8b, NAME => 'test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-11T16:15:04,410 INFO [StoreOpener-e762741692724ea54cb4ce74a7e615ae-1 {}] regionserver.HStore(327): Store=e762741692724ea54cb4ce74a7e615ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,411 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,411 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,412 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,412 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:04,412 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:04,412 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,412 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d939495820cb0d73d77301eb7bbcb7fb, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:04,412 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => f9f064481e2ac446aeb716014e0cdf6b, NAME => 'test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,413 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,415 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,416 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,416 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,417 INFO [StoreOpener-7df81ce6daca1d892d80d840de0c0b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,420 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,428 INFO [StoreOpener-7df81ce6daca1d892d80d840de0c0b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7df81ce6daca1d892d80d840de0c0b8b columnFamilyName family 2024-11-11T16:15:04,429 DEBUG [StoreOpener-7df81ce6daca1d892d80d840de0c0b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=23 2024-11-11T16:15:04,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure d939495820cb0d73d77301eb7bbcb7fb, server=16b413a53992,33497,1731341698716 in 286 msec 2024-11-11T16:15:04,431 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e762741692724ea54cb4ce74a7e615ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,434 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened e762741692724ea54cb4ce74a7e615ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74431303, jitterRate=0.10911284387111664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,434 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:04,434 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for e762741692724ea54cb4ce74a7e615ae: Running coprocessor pre-open hook at 1731341704390Writing region info on filesystem at 1731341704390Initializing all the Stores at 1731341704391 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704392 (+1 ms)Cleaning up temporary data from old regions at 1731341704417 (+25 ms)Running coprocessor post-open hooks at 1731341704434 (+17 ms)Region opened successfully at 1731341704434 2024-11-11T16:15:04,436 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae., pid=40, masterSystemTime=1731341704282 2024-11-11T16:15:04,437 INFO [StoreOpener-7df81ce6daca1d892d80d840de0c0b8b-1 {}] regionserver.HStore(327): Store=7df81ce6daca1d892d80d840de0c0b8b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,437 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d939495820cb0d73d77301eb7bbcb7fb, ASSIGN in 678 msec 2024-11-11T16:15:04,437 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,439 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,440 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,442 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,442 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9f064481e2ac446aeb716014e0cdf6b columnFamilyName family 2024-11-11T16:15:04,442 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,442 DEBUG [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=7ee84052923dbaf7f5c9581236276c4c, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,443 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] regionserver.HStore(327): Store=f9f064481e2ac446aeb716014e0cdf6b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,443 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:04,443 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:04,443 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,444 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:04,444 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 98ffcdd491bf651a96a2bf2c2b5a9a95, NAME => 'test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-11T16:15:04,444 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,445 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,445 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,445 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,445 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,446 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,446 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,448 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,448 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,451 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=30 2024-11-11T16:15:04,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,36245,1731341698838 in 375 msec 2024-11-11T16:15:04,459 INFO [StoreOpener-98ffcdd491bf651a96a2bf2c2b5a9a95-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,462 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7df81ce6daca1d892d80d840de0c0b8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,464 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 7df81ce6daca1d892d80d840de0c0b8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72633859, jitterRate=0.08232884109020233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,464 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:04,464 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 7df81ce6daca1d892d80d840de0c0b8b: Running coprocessor pre-open hook at 1731341704413Writing region info on filesystem at 1731341704413Initializing all the Stores at 1731341704415 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704415Cleaning up temporary data from old regions at 1731341704442 (+27 ms)Running coprocessor post-open hooks at 1731341704464 (+22 ms)Region opened successfully at 1731341704464 2024-11-11T16:15:04,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN in 707 msec 2024-11-11T16:15:04,466 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b., pid=35, masterSystemTime=1731341704168 2024-11-11T16:15:04,471 INFO [StoreOpener-98ffcdd491bf651a96a2bf2c2b5a9a95-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98ffcdd491bf651a96a2bf2c2b5a9a95 columnFamilyName family 2024-11-11T16:15:04,471 DEBUG [StoreOpener-98ffcdd491bf651a96a2bf2c2b5a9a95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,472 INFO [StoreOpener-98ffcdd491bf651a96a2bf2c2b5a9a95-1 {}] regionserver.HStore(327): Store=98ffcdd491bf651a96a2bf2c2b5a9a95/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,472 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,473 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/f9f064481e2ac446aeb716014e0cdf6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,474 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,475 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened f9f064481e2ac446aeb716014e0cdf6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65066847, jitterRate=-0.03042842447757721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,475 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:04,475 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for f9f064481e2ac446aeb716014e0cdf6b: Running coprocessor pre-open hook at 1731341704414Writing region info on filesystem at 1731341704414Initializing all the Stores at 1731341704415 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704415Cleaning up temporary data from old regions at 1731341704448 (+33 ms)Running coprocessor post-open hooks at 1731341704475 (+27 ms)Region opened successfully at 1731341704475 2024-11-11T16:15:04,475 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:04,475 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:04,476 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,478 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,478 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,478 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b., pid=42, masterSystemTime=1731341704252 2024-11-11T16:15:04,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e762741692724ea54cb4ce74a7e615ae, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:04,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,483 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,486 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:04,486 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,44579,1731341698497, table=test, region=7df81ce6daca1d892d80d840de0c0b8b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,486 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:04,486 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:04,486 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 9984848611dae4de6b309aeff78ed621, NAME => 'test,www,1731341702648.9984848611dae4de6b309aeff78ed621.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-11T16:15:04,487 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,487 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,www,1731341702648.9984848611dae4de6b309aeff78ed621.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,487 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,487 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7df81ce6daca1d892d80d840de0c0b8b, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:04,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,497 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,503 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9984848611dae4de6b309aeff78ed621 columnFamilyName family 2024-11-11T16:15:04,504 DEBUG [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,504 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] regionserver.HStore(327): Store=9984848611dae4de6b309aeff78ed621/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,505 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,506 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,506 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,507 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,507 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,512 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,518 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/98ffcdd491bf651a96a2bf2c2b5a9a95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,526 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,33497,1731341698716, table=test, region=e762741692724ea54cb4ce74a7e615ae. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,528 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 98ffcdd491bf651a96a2bf2c2b5a9a95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68904834, jitterRate=0.026762038469314575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,528 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:04,528 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 98ffcdd491bf651a96a2bf2c2b5a9a95: Running coprocessor pre-open hook at 1731341704445Writing region info on filesystem at 1731341704445Initializing all the Stores at 1731341704447 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704448 (+1 ms)Cleaning up temporary data from old regions at 1731341704478 (+30 ms)Running coprocessor post-open hooks at 1731341704528 (+50 ms)Region opened successfully at 1731341704528 2024-11-11T16:15:04,529 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,531 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,36245,1731341698838, table=test, region=f9f064481e2ac446aeb716014e0cdf6b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,531 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95., pid=38, masterSystemTime=1731341704282 2024-11-11T16:15:04,532 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 9984848611dae4de6b309aeff78ed621; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68435770, jitterRate=0.019772440195083618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,532 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:04,533 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 9984848611dae4de6b309aeff78ed621: Running coprocessor pre-open hook at 1731341704487Writing region info on filesystem at 1731341704487Initializing all the Stores at 1731341704490 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704490Cleaning up temporary data from old regions at 1731341704507 (+17 ms)Running coprocessor post-open hooks at 1731341704532 (+25 ms)Region opened successfully at 1731341704532 2024-11-11T16:15:04,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=18 2024-11-11T16:15:04,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 7df81ce6daca1d892d80d840de0c0b8b, server=16b413a53992,44579,1731341698497 in 445 msec 2024-11-11T16:15:04,536 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731341702648.9984848611dae4de6b309aeff78ed621., pid=43, masterSystemTime=1731341704252 2024-11-11T16:15:04,538 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:04,538 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:04,538 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:04,538 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e62b8abd36e22d5d589c5bdd191ed105, NAME => 'test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-11T16:15:04,539 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,539 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,539 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,539 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,544 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:04,544 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:04,544 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:04,544 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 97c9a93ca402a988eed0542bbe16f17a, NAME => 'test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-11T16:15:04,545 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,545 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,545 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,545 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7df81ce6daca1d892d80d840de0c0b8b, ASSIGN in 783 msec 2024-11-11T16:15:04,555 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9984848611dae4de6b309aeff78ed621, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,557 INFO [StoreOpener-e62b8abd36e22d5d589c5bdd191ed105-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,565 INFO [StoreOpener-e62b8abd36e22d5d589c5bdd191ed105-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e62b8abd36e22d5d589c5bdd191ed105 columnFamilyName family 2024-11-11T16:15:04,565 DEBUG [StoreOpener-e62b8abd36e22d5d589c5bdd191ed105-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98ffcdd491bf651a96a2bf2c2b5a9a95, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:04,570 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,572 INFO [StoreOpener-e62b8abd36e22d5d589c5bdd191ed105-1 {}] regionserver.HStore(327): Store=e62b8abd36e22d5d589c5bdd191ed105/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,572 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,573 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97c9a93ca402a988eed0542bbe16f17a columnFamilyName family 2024-11-11T16:15:04,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=20 2024-11-11T16:15:04,573 DEBUG [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,36245,1731341698838 in 408 msec 2024-11-11T16:15:04,574 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,574 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] regionserver.HStore(327): Store=97c9a93ca402a988eed0542bbe16f17a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,575 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,575 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,575 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,575 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,576 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,577 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,577 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,577 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN in 822 msec 2024-11-11T16:15:04,580 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,583 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=19 2024-11-11T16:15:04,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 98ffcdd491bf651a96a2bf2c2b5a9a95, server=16b413a53992,33497,1731341698716 in 456 msec 2024-11-11T16:15:04,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=98ffcdd491bf651a96a2bf2c2b5a9a95, ASSIGN in 834 msec 2024-11-11T16:15:04,600 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e62b8abd36e22d5d589c5bdd191ed105/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,603 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e62b8abd36e22d5d589c5bdd191ed105; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66638748, jitterRate=-0.007005274295806885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:04,603 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e62b8abd36e22d5d589c5bdd191ed105: Running coprocessor pre-open hook at 1731341704540Writing region info on filesystem at 1731341704540Initializing all the Stores at 1731341704552 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704552Cleaning up temporary data from old regions at 1731341704575 (+23 ms)Running coprocessor post-open hooks at 1731341704603 (+28 ms)Region opened successfully at 1731341704603 2024-11-11T16:15:04,607 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105., pid=41, masterSystemTime=1731341704282 2024-11-11T16:15:04,611 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/97c9a93ca402a988eed0542bbe16f17a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=27 2024-11-11T16:15:04,612 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:04,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,36245,1731341698838 in 432 msec 2024-11-11T16:15:04,612 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:04,612 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 97c9a93ca402a988eed0542bbe16f17a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60992474, jitterRate=-0.09114131331443787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,612 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:04,612 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 97c9a93ca402a988eed0542bbe16f17a: Running coprocessor pre-open hook at 1731341704545Writing region info on filesystem at 1731341704545Initializing all the Stores at 1731341704565 (+20 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704565Cleaning up temporary data from old regions at 1731341704577 (+12 ms)Running coprocessor post-open hooks at 1731341704612 (+35 ms)Region opened successfully at 1731341704612 2024-11-11T16:15:04,614 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a., pid=44, masterSystemTime=1731341704519 2024-11-11T16:15:04,618 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:04,619 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:04,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN in 860 msec 2024-11-11T16:15:04,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=97c9a93ca402a988eed0542bbe16f17a, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,632 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,33497,1731341698716, table=test, region=e762741692724ea54cb4ce74a7e615ae. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,636 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,36245,1731341698838, table=test, region=97c9a93ca402a988eed0542bbe16f17a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:04,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e62b8abd36e22d5d589c5bdd191ed105, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:04,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=8 2024-11-11T16:15:04,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,36245,1731341698838 in 294 msec 2024-11-11T16:15:04,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN in 914 msec 2024-11-11T16:15:04,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=29 2024-11-11T16:15:04,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e62b8abd36e22d5d589c5bdd191ed105, server=16b413a53992,33497,1731341698716 in 522 msec 2024-11-11T16:15:04,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e62b8abd36e22d5d589c5bdd191ed105, ASSIGN in 926 msec 2024-11-11T16:15:04,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=25 2024-11-11T16:15:04,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e762741692724ea54cb4ce74a7e615ae, server=16b413a53992,33497,1731341698716 in 541 msec 2024-11-11T16:15:04,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e762741692724ea54cb4ce74a7e615ae, ASSIGN in 941 msec 2024-11-11T16:15:04,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:04,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fb28854349ec0d596f8f0dca65b13b8, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:04,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-11T16:15:04,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fb28854349ec0d596f8f0dca65b13b8, server=16b413a53992,44579,1731341698497 in 729 msec 2024-11-11T16:15:04,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7fb28854349ec0d596f8f0dca65b13b8, ASSIGN in 982 msec 2024-11-11T16:15:04,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:04,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=24 2024-11-11T16:15:04,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838 in 669 msec 2024-11-11T16:15:04,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, ASSIGN in 1.0400 sec 2024-11-11T16:15:04,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f2e670caa2864b9154b87162f54c642, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8845a47073d69ee0a0021d18e08e51ef, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba7b9cc976155fbb76bf00c2ffd4495, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:04,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=4f42aec7889fcb07b96681cea332b42d, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=756a2cfebbc98a6396c782a802598ffc, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,836 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:04,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=62d867b8a89d21a5e2538e1e5222da88, ASSIGN 2024-11-11T16:15:04,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f42aec7889fcb07b96681cea332b42d, ASSIGN because future has completed 2024-11-11T16:15:04,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd1d8bee83938f0837b5cdacde7e89c4, ASSIGN because future has completed 2024-11-11T16:15:04,841 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=62d867b8a89d21a5e2538e1e5222da88, ASSIGN; state=OFFLINE, location=16b413a53992,44579,1731341698497; forceNewPlan=false, retain=false 2024-11-11T16:15:04,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=756a2cfebbc98a6396c782a802598ffc, ASSIGN because future has completed 2024-11-11T16:15:04,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd1d8bee83938f0837b5cdacde7e89c4, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d05687468bd618dc7fa99d7c6b5f3b2, ASSIGN because future has completed 2024-11-11T16:15:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:04,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d05687468bd618dc7fa99d7c6b5f3b2, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,873 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:04,873 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c93273fb8e84a5a8b730fc9fafb6f7, NAME => 'test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-11T16:15:04,873 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,873 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,874 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,874 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 756a2cfebbc98a6396c782a802598ffc, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,882 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,885 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c93273fb8e84a5a8b730fc9fafb6f7 columnFamilyName family 2024-11-11T16:15:04,886 DEBUG [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,888 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] regionserver.HStore(327): Store=58c93273fb8e84a5a8b730fc9fafb6f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,888 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,890 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,891 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,892 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,892 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,895 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f42aec7889fcb07b96681cea332b42d, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:04,916 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,918 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 58c93273fb8e84a5a8b730fc9fafb6f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71358671, jitterRate=0.06332705914974213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,918 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:04,919 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 58c93273fb8e84a5a8b730fc9fafb6f7: Running coprocessor pre-open hook at 1731341704874Writing region info on filesystem at 1731341704874Initializing all the Stores at 1731341704876 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704876Cleaning up temporary data from old regions at 1731341704892 (+16 ms)Running coprocessor post-open hooks at 1731341704919 (+27 ms)Region opened successfully at 1731341704919 2024-11-11T16:15:04,920 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7., pid=48, masterSystemTime=1731341704861 2024-11-11T16:15:04,924 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:04,924 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:04,924 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:04,925 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => aebcd9c83edd713d66ccd5d8f1a5150d, NAME => 'test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-11T16:15:04,925 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,925 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,925 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,925 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:04,936 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,943 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebcd9c83edd713d66ccd5d8f1a5150d columnFamilyName family 2024-11-11T16:15:04,943 DEBUG [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=17 2024-11-11T16:15:04,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,36245,1731341698838 in 167 msec 2024-11-11T16:15:04,945 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] regionserver.HStore(327): Store=aebcd9c83edd713d66ccd5d8f1a5150d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:04,945 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,946 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,948 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN in 1.1920 sec 2024-11-11T16:15:04,949 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,949 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,955 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,980 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:04,981 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened aebcd9c83edd713d66ccd5d8f1a5150d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65465980, jitterRate=-0.024480879306793213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:04,981 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:04,982 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for aebcd9c83edd713d66ccd5d8f1a5150d: Running coprocessor pre-open hook at 1731341704926Writing region info on filesystem at 1731341704926Initializing all the Stores at 1731341704930 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704930Cleaning up temporary data from old regions at 1731341704949 (+19 ms)Running coprocessor post-open hooks at 1731341704982 (+33 ms)Region opened successfully at 1731341704982 2024-11-11T16:15:04,983 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d., pid=45, masterSystemTime=1731341704861 2024-11-11T16:15:04,986 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:04,986 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f2e670caa2864b9154b87162f54c642, NAME => 'test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-11T16:15:04,987 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:04,987 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,987 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:04,987 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:04,992 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:04,992 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:04,992 INFO [16b413a53992:35895 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T16:15:04,992 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:04,992 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => b7e69cf109199b3c3d754201ea9818cd, NAME => 'test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-11T16:15:04,993 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:04,993 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:04,993 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:04,993 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:04,994 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:04,994 INFO [StoreOpener-5f2e670caa2864b9154b87162f54c642-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=62d867b8a89d21a5e2538e1e5222da88, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:05,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:05,013 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,014 INFO [StoreOpener-5f2e670caa2864b9154b87162f54c642-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f2e670caa2864b9154b87162f54c642 columnFamilyName family 2024-11-11T16:15:05,014 DEBUG [StoreOpener-5f2e670caa2864b9154b87162f54c642-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=62d867b8a89d21a5e2538e1e5222da88, ASSIGN because future has completed 2024-11-11T16:15:05,023 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e69cf109199b3c3d754201ea9818cd columnFamilyName family 2024-11-11T16:15:05,023 DEBUG [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=7 2024-11-11T16:15:05,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,36245,1731341698838 in 307 msec 2024-11-11T16:15:05,025 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] regionserver.HStore(327): Store=b7e69cf109199b3c3d754201ea9818cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,026 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,027 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN in 1.2720 sec 2024-11-11T16:15:05,027 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,028 INFO [StoreOpener-5f2e670caa2864b9154b87162f54c642-1 {}] regionserver.HStore(327): Store=5f2e670caa2864b9154b87162f54c642/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,028 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,028 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62d867b8a89d21a5e2538e1e5222da88, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:05,030 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,030 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:05,030 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f42aec7889fcb07b96681cea332b42d, NAME => 'test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-11T16:15:05,030 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,030 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,031 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,031 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,031 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,031 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,031 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,032 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,032 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,036 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,044 INFO [StoreOpener-4f42aec7889fcb07b96681cea332b42d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,050 INFO [StoreOpener-4f42aec7889fcb07b96681cea332b42d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f42aec7889fcb07b96681cea332b42d columnFamilyName family 2024-11-11T16:15:05,050 DEBUG [StoreOpener-4f42aec7889fcb07b96681cea332b42d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,052 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,053 INFO [StoreOpener-4f42aec7889fcb07b96681cea332b42d-1 {}] regionserver.HStore(327): Store=4f42aec7889fcb07b96681cea332b42d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,053 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,053 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened b7e69cf109199b3c3d754201ea9818cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66870940, jitterRate=-0.0035453438758850098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,053 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:05,054 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for b7e69cf109199b3c3d754201ea9818cd: Running coprocessor pre-open hook at 1731341704993Writing region info on filesystem at 1731341704993Initializing all the Stores at 1731341704999 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704999Cleaning up temporary data from old regions at 1731341705028 (+29 ms)Running coprocessor post-open hooks at 1731341705053 (+25 ms)Region opened successfully at 1731341705054 (+1 ms) 2024-11-11T16:15:05,054 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,055 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,056 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5f2e670caa2864b9154b87162f54c642/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,058 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,058 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,058 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5f2e670caa2864b9154b87162f54c642; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61269996, jitterRate=-0.08700591325759888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,058 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:05,058 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5f2e670caa2864b9154b87162f54c642: Running coprocessor pre-open hook at 1731341704987Writing region info on filesystem at 1731341704987Initializing all the Stores at 1731341704991 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341704991Cleaning up temporary data from old regions at 1731341705032 (+41 ms)Running coprocessor post-open hooks at 1731341705058 (+26 ms)Region opened successfully at 1731341705058 2024-11-11T16:15:05,059 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd., pid=46, masterSystemTime=1731341704861 2024-11-11T16:15:05,062 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,064 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642., pid=49, masterSystemTime=1731341704969 2024-11-11T16:15:05,064 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:05,064 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:05,064 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:05,065 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 5317ee89294e310637e2785c5a255b6b, NAME => 'test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-11T16:15:05,065 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,065 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,065 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,065 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b7e69cf109199b3c3d754201ea9818cd, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:05,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:05,072 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:05,072 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:05,073 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:05,074 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ba7b9cc976155fbb76bf00c2ffd4495, NAME => 'test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-11T16:15:05,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:05,076 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,076 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,077 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,077 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f2e670caa2864b9154b87162f54c642, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:05,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-11-11T16:15:05,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,36245,1731341698838 in 332 msec 2024-11-11T16:15:05,085 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4f42aec7889fcb07b96681cea332b42d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,086 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,086 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 4f42aec7889fcb07b96681cea332b42d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67156609, jitterRate=7.114559412002563E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,087 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:05,087 INFO [StoreOpener-7ba7b9cc976155fbb76bf00c2ffd4495-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,087 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 4f42aec7889fcb07b96681cea332b42d: Running coprocessor pre-open hook at 1731341705031Writing region info on filesystem at 1731341705031Initializing all the Stores at 1731341705032 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705033 (+1 ms)Cleaning up temporary data from old regions at 1731341705058 (+25 ms)Running coprocessor post-open hooks at 1731341705087 (+29 ms)Region opened successfully at 1731341705087 2024-11-11T16:15:05,090 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d., pid=55, masterSystemTime=1731341705013 2024-11-11T16:15:05,090 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5317ee89294e310637e2785c5a255b6b columnFamilyName family 2024-11-11T16:15:05,090 DEBUG [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,091 INFO [StoreOpener-7ba7b9cc976155fbb76bf00c2ffd4495-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ba7b9cc976155fbb76bf00c2ffd4495 columnFamilyName family 2024-11-11T16:15:05,091 DEBUG [StoreOpener-7ba7b9cc976155fbb76bf00c2ffd4495-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,091 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] regionserver.HStore(327): Store=5317ee89294e310637e2785c5a255b6b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,092 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,094 INFO [StoreOpener-7ba7b9cc976155fbb76bf00c2ffd4495-1 {}] regionserver.HStore(327): Store=7ba7b9cc976155fbb76bf00c2ffd4495/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,094 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,094 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,094 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:05,095 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,095 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d05687468bd618dc7fa99d7c6b5f3b2, NAME => 'test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,095 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,096 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN in 1.3310 sec 2024-11-11T16:15:05,099 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,099 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:05,100 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,102 INFO [StoreOpener-6d05687468bd618dc7fa99d7c6b5f3b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,108 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ba7b9cc976155fbb76bf00c2ffd4495/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,110 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 7ba7b9cc976155fbb76bf00c2ffd4495; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69254929, jitterRate=0.03197886049747467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,110 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:05,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f42aec7889fcb07b96681cea332b42d, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:05,110 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 7ba7b9cc976155fbb76bf00c2ffd4495: Running coprocessor pre-open hook at 1731341705077Writing region info on filesystem at 1731341705077Initializing all the Stores at 1731341705080 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705080Cleaning up temporary data from old regions at 1731341705096 (+16 ms)Running coprocessor post-open hooks at 1731341705110 (+14 ms)Region opened successfully at 1731341705110 2024-11-11T16:15:05,111 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495., pid=51, masterSystemTime=1731341704969 2024-11-11T16:15:05,118 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,118 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:05,119 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:05,119 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:05,119 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 5317ee89294e310637e2785c5a255b6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66905028, jitterRate=-0.003037393093109131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,119 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 8845a47073d69ee0a0021d18e08e51ef, NAME => 'test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-11T16:15:05,119 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:05,119 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 5317ee89294e310637e2785c5a255b6b: Running coprocessor pre-open hook at 1731341705066Writing region info on filesystem at 1731341705066Initializing all the Stores at 1731341705071 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705071Cleaning up temporary data from old regions at 1731341705095 (+24 ms)Running coprocessor post-open hooks at 1731341705119 (+24 ms)Region opened successfully at 1731341705119 2024-11-11T16:15:05,119 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,120 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,120 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,120 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,120 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,44579,1731341698497, table=test, region=5f2e670caa2864b9154b87162f54c642. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:05,123 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b., pid=47, masterSystemTime=1731341704861 2024-11-11T16:15:05,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-11T16:15:05,128 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:05,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f2e670caa2864b9154b87162f54c642, server=16b413a53992,44579,1731341698497 in 322 msec 2024-11-11T16:15:05,129 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:05,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5f2e670caa2864b9154b87162f54c642, ASSIGN in 1.3770 sec 2024-11-11T16:15:05,133 INFO [StoreOpener-8845a47073d69ee0a0021d18e08e51ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5317ee89294e310637e2785c5a255b6b, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:05,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:05,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:05,144 INFO [StoreOpener-6d05687468bd618dc7fa99d7c6b5f3b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d05687468bd618dc7fa99d7c6b5f3b2 columnFamilyName family 2024-11-11T16:15:05,144 DEBUG [StoreOpener-6d05687468bd618dc7fa99d7c6b5f3b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,145 INFO [StoreOpener-8845a47073d69ee0a0021d18e08e51ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8845a47073d69ee0a0021d18e08e51ef columnFamilyName family 2024-11-11T16:15:05,145 DEBUG [StoreOpener-8845a47073d69ee0a0021d18e08e51ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba7b9cc976155fbb76bf00c2ffd4495, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:05,147 INFO [StoreOpener-8845a47073d69ee0a0021d18e08e51ef-1 {}] regionserver.HStore(327): Store=8845a47073d69ee0a0021d18e08e51ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,147 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,148 INFO [StoreOpener-6d05687468bd618dc7fa99d7c6b5f3b2-1 {}] regionserver.HStore(327): Store=6d05687468bd618dc7fa99d7c6b5f3b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,148 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,149 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,151 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,151 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,154 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,155 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,155 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,33497,1731341698716, table=test, region=4f42aec7889fcb07b96681cea332b42d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:05,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=14 2024-11-11T16:15:05,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,36245,1731341698838 in 393 msec 2024-11-11T16:15:05,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN in 1.4100 sec 2024-11-11T16:15:05,168 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/8845a47073d69ee0a0021d18e08e51ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,169 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 8845a47073d69ee0a0021d18e08e51ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59241892, jitterRate=-0.11722701787948608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,170 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:05,170 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 8845a47073d69ee0a0021d18e08e51ef: Running coprocessor pre-open hook at 1731341705120Writing region info on filesystem at 1731341705120Initializing all the Stores at 1731341705122 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705122Cleaning up temporary data from old regions at 1731341705150 (+28 ms)Running coprocessor post-open hooks at 1731341705170 (+20 ms)Region opened successfully at 1731341705170 2024-11-11T16:15:05,172 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef., pid=50, masterSystemTime=1731341704969 2024-11-11T16:15:05,177 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:05,177 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:05,177 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/6d05687468bd618dc7fa99d7c6b5f3b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,178 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 6d05687468bd618dc7fa99d7c6b5f3b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61877027, jitterRate=-0.07796044647693634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,178 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:05,178 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 6d05687468bd618dc7fa99d7c6b5f3b2: Running coprocessor pre-open hook at 1731341705096Writing region info on filesystem at 1731341705096Initializing all the Stores at 1731341705098 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705098Cleaning up temporary data from old regions at 1731341705151 (+53 ms)Running coprocessor post-open hooks at 1731341705178 (+27 ms)Region opened successfully at 1731341705178 2024-11-11T16:15:05,180 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2., pid=53, masterSystemTime=1731341705013 2024-11-11T16:15:05,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=15 2024-11-11T16:15:05,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ba7b9cc976155fbb76bf00c2ffd4495, server=16b413a53992,44579,1731341698497 in 344 msec 2024-11-11T16:15:05,183 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:05,183 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:05,183 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:05,184 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => cd1d8bee83938f0837b5cdacde7e89c4, NAME => 'test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-11T16:15:05,184 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,184 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,184 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,184 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,189 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:05,189 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 62d867b8a89d21a5e2538e1e5222da88, NAME => 'test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-11T16:15:05,189 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,189 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,190 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,190 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:05,193 INFO [StoreOpener-cd1d8bee83938f0837b5cdacde7e89c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d05687468bd618dc7fa99d7c6b5f3b2, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:05,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ba7b9cc976155fbb76bf00c2ffd4495, ASSIGN in 1.4440 sec 2024-11-11T16:15:05,203 INFO [StoreOpener-cd1d8bee83938f0837b5cdacde7e89c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd1d8bee83938f0837b5cdacde7e89c4 columnFamilyName family 2024-11-11T16:15:05,206 INFO [StoreOpener-62d867b8a89d21a5e2538e1e5222da88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,207 DEBUG [StoreOpener-cd1d8bee83938f0837b5cdacde7e89c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,209 INFO [StoreOpener-62d867b8a89d21a5e2538e1e5222da88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62d867b8a89d21a5e2538e1e5222da88 columnFamilyName family 2024-11-11T16:15:05,209 DEBUG [StoreOpener-62d867b8a89d21a5e2538e1e5222da88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,210 INFO [StoreOpener-62d867b8a89d21a5e2538e1e5222da88-1 {}] regionserver.HStore(327): Store=62d867b8a89d21a5e2538e1e5222da88/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,210 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,211 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,212 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,212 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,212 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,215 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,219 INFO [StoreOpener-cd1d8bee83938f0837b5cdacde7e89c4-1 {}] regionserver.HStore(327): Store=cd1d8bee83938f0837b5cdacde7e89c4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,220 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,221 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,221 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,222 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,222 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,224 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/62d867b8a89d21a5e2538e1e5222da88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,225 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 62d867b8a89d21a5e2538e1e5222da88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60230228, jitterRate=-0.10249966382980347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,225 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:05,226 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 62d867b8a89d21a5e2538e1e5222da88: Running coprocessor pre-open hook at 1731341705190Writing region info on filesystem at 1731341705190Initializing all the Stores at 1731341705191 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705191Cleaning up temporary data from old regions at 1731341705212 (+21 ms)Running coprocessor post-open hooks at 1731341705225 (+13 ms)Region opened successfully at 1731341705226 (+1 ms) 2024-11-11T16:15:05,226 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,227 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88., pid=56, masterSystemTime=1731341705182 2024-11-11T16:15:05,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=6 2024-11-11T16:15:05,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d05687468bd618dc7fa99d7c6b5f3b2, server=16b413a53992,33497,1731341698716 in 337 msec 2024-11-11T16:15:05,241 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:05,242 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:05,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d05687468bd618dc7fa99d7c6b5f3b2, ASSIGN in 1.4760 sec 2024-11-11T16:15:05,248 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/cd1d8bee83938f0837b5cdacde7e89c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:05,249 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened cd1d8bee83938f0837b5cdacde7e89c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61320527, jitterRate=-0.08625294268131256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,249 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:05,249 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for cd1d8bee83938f0837b5cdacde7e89c4: Running coprocessor pre-open hook at 1731341705184Writing region info on filesystem at 1731341705184Initializing all the Stores at 1731341705188 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705188Cleaning up temporary data from old regions at 1731341705222 (+34 ms)Running coprocessor post-open hooks at 1731341705249 (+27 ms)Region opened successfully at 1731341705249 2024-11-11T16:15:05,252 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4., pid=52, masterSystemTime=1731341705013 2024-11-11T16:15:05,256 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:05,257 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:05,257 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:05,257 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 756a2cfebbc98a6396c782a802598ffc, NAME => 'test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-11T16:15:05,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62d867b8a89d21a5e2538e1e5222da88, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:05,257 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,258 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:05,258 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,258 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,260 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,33497,1731341698716, table=test, region=4f42aec7889fcb07b96681cea332b42d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:05,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:05,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8845a47073d69ee0a0021d18e08e51ef, server=16b413a53992,44579,1731341698497 because future has completed 2024-11-11T16:15:05,272 INFO [StoreOpener-756a2cfebbc98a6396c782a802598ffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,275 INFO [StoreOpener-756a2cfebbc98a6396c782a802598ffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756a2cfebbc98a6396c782a802598ffc columnFamilyName family 2024-11-11T16:15:05,275 DEBUG [StoreOpener-756a2cfebbc98a6396c782a802598ffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,276 INFO [StoreOpener-756a2cfebbc98a6396c782a802598ffc-1 {}] regionserver.HStore(327): Store=756a2cfebbc98a6396c782a802598ffc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:05,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-11T16:15:05,276 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 62d867b8a89d21a5e2538e1e5222da88, server=16b413a53992,44579,1731341698497 in 239 msec 2024-11-11T16:15:05,277 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,277 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,278 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,278 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=62d867b8a89d21a5e2538e1e5222da88, ASSIGN in 1.5240 sec 2024-11-11T16:15:05,281 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-11-11T16:15:05,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 8845a47073d69ee0a0021d18e08e51ef, server=16b413a53992,44579,1731341698497 in 458 msec 2024-11-11T16:15:05,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8845a47073d69ee0a0021d18e08e51ef, ASSIGN in 1.5320 sec 2024-11-11T16:15:05,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:05,293 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/756a2cfebbc98a6396c782a802598ffc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:05,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd1d8bee83938f0837b5cdacde7e89c4, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:05,294 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 756a2cfebbc98a6396c782a802598ffc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65138813, jitterRate=-0.02935604751110077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:05,294 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:05,294 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 756a2cfebbc98a6396c782a802598ffc: Running coprocessor pre-open hook at 1731341705258Writing region info on filesystem at 1731341705258Initializing all the Stores at 1731341705260 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341705260Cleaning up temporary data from old regions at 1731341705278 (+18 ms)Running coprocessor post-open hooks at 1731341705294 (+16 ms)Region opened successfully at 1731341705294 2024-11-11T16:15:05,296 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc., pid=54, masterSystemTime=1731341705013 2024-11-11T16:15:05,299 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:05,299 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:05,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=16 2024-11-11T16:15:05,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f42aec7889fcb07b96681cea332b42d, server=16b413a53992,33497,1731341698716 in 390 msec 2024-11-11T16:15:05,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4f42aec7889fcb07b96681cea332b42d, ASSIGN in 1.5480 sec 2024-11-11T16:15:05,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:05,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 756a2cfebbc98a6396c782a802598ffc, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:05,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=13 2024-11-11T16:15:05,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure cd1d8bee83938f0837b5cdacde7e89c4, server=16b413a53992,33497,1731341698716 in 465 msec 2024-11-11T16:15:05,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cd1d8bee83938f0837b5cdacde7e89c4, ASSIGN in 1.5850 sec 2024-11-11T16:15:05,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=11 2024-11-11T16:15:05,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 756a2cfebbc98a6396c782a802598ffc, server=16b413a53992,33497,1731341698716 in 473 msec 2024-11-11T16:15:05,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=4 2024-11-11T16:15:05,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=756a2cfebbc98a6396c782a802598ffc, ASSIGN in 1.6030 sec 2024-11-11T16:15:05,367 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:15:05,367 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341705367"}]},"ts":"1731341705367"} 2024-11-11T16:15:05,372 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-11T16:15:05,375 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:15:05,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.7100 sec 2024-11-11T16:15:06,746 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-11T16:15:06,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T16:15:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:15:06,880 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-11T16:15:06,880 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-11T16:15:06,882 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:06,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-11T16:15:06,943 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:06,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-11T16:15:06,944 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:06,957 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-11T16:15:06,996 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-11T16:15:07,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:07,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:07,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:07,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d05687468bd618dc7fa99d7c6b5f3b2 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 756a2cfebbc98a6396c782a802598ffc changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cd1d8bee83938f0837b5cdacde7e89c4 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4f42aec7889fcb07b96681cea332b42d changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 98ffcdd491bf651a96a2bf2c2b5a9a95 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d939495820cb0d73d77301eb7bbcb7fb changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e762741692724ea54cb4ce74a7e615ae changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e62b8abd36e22d5d589c5bdd191ed105 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aebcd9c83edd713d66ccd5d8f1a5150d changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97c9a93ca402a988eed0542bbe16f17a changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b7e69cf109199b3c3d754201ea9818cd changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5317ee89294e310637e2785c5a255b6b changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 58c93273fb8e84a5a8b730fc9fafb6f7 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f9f064481e2ac446aeb716014e0cdf6b changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 60aabcb1be5df802409ef5ef8a1eccba changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9984848611dae4de6b309aeff78ed621 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7ee84052923dbaf7f5c9581236276c4c changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 62d867b8a89d21a5e2538e1e5222da88 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f2e670caa2864b9154b87162f54c642 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8845a47073d69ee0a0021d18e08e51ef changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7ba7b9cc976155fbb76bf00c2ffd4495 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7df81ce6daca1d892d80d840de0c0b8b changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 35b42287bbbc0c5919124a8c4b800ea1 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e3cef8c6c3398e017e8dfa5cd0c9c242 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4c2941f7d018cc329ed386329a479fbf changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7fb28854349ec0d596f8f0dca65b13b8 changed from -1.0 to 0.0, refreshing cache 2024-11-11T16:15:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-11T16:15:07,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-11T16:15:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-11T16:15:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(204): Hosts are {16b413a53992=0} racks are {/default-rack=0} 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T16:15:07,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T16:15:07,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T16:15:07,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T16:15:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-11T16:15:07,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-11T16:15:07,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-11T16:15:07,445 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:15:07,989 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T16:15:07,990 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T16:15:07,991 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-11T16:15:07,991 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-11T16:15:07,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T16:15:07,993 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T16:15:07,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T16:15:07,993 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T16:15:07,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 785 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-11T16:15:08,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-11T16:15:08,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster(2172): balance hri=60aabcb1be5df802409ef5ef8a1eccba, source=16b413a53992,36245,1731341698838, destination=16b413a53992,33497,1731341698716 2024-11-11T16:15:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:08,004 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:08,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=CLOSING, regionLocation=16b413a53992,36245,1731341698838 2024-11-11T16:15:08,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE because future has completed 2024-11-11T16:15:08,010 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:15:08,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838}] 2024-11-11T16:15:08,168 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,168 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:15:08,169 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 60aabcb1be5df802409ef5ef8a1eccba, disabling compactions & flushes 2024-11-11T16:15:08,169 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,169 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,169 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. after waiting 0 ms 2024-11-11T16:15:08,169 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,175 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:08,178 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,178 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 60aabcb1be5df802409ef5ef8a1eccba: Waiting for close lock at 1731341708169Running coprocessor pre-close hooks at 1731341708169Disabling compacts and flushes for region at 1731341708169Disabling writes for close at 1731341708169Writing region close event to WAL at 1731341708170 (+1 ms)Running coprocessor post-close hooks at 1731341708176 (+6 ms)Closed at 1731341708178 (+2 ms) 2024-11-11T16:15:08,179 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 60aabcb1be5df802409ef5ef8a1eccba move to 16b413a53992,33497,1731341698716 record at close sequenceid=2 2024-11-11T16:15:08,182 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,183 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=CLOSED 2024-11-11T16:15:08,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838 because future has completed 2024-11-11T16:15:08,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T16:15:08,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,36245,1731341698838 in 178 msec 2024-11-11T16:15:08,197 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE; state=CLOSED, location=16b413a53992,33497,1731341698716; forceNewPlan=false, retain=false 2024-11-11T16:15:08,347 INFO [16b413a53992:35895 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T16:15:08,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:08,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE because future has completed 2024-11-11T16:15:08,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:08,509 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,509 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 60aabcb1be5df802409ef5ef8a1eccba, NAME => 'test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-11T16:15:08,510 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,510 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:08,510 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,510 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,512 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,513 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60aabcb1be5df802409ef5ef8a1eccba columnFamilyName family 2024-11-11T16:15:08,513 DEBUG [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:08,514 INFO [StoreOpener-60aabcb1be5df802409ef5ef8a1eccba-1 {}] regionserver.HStore(327): Store=60aabcb1be5df802409ef5ef8a1eccba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:08,514 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,515 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,517 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,517 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,517 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,520 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,521 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 60aabcb1be5df802409ef5ef8a1eccba; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64574671, jitterRate=-0.037762418389320374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:08,521 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:08,521 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 60aabcb1be5df802409ef5ef8a1eccba: Running coprocessor pre-open hook at 1731341708510Writing region info on filesystem at 1731341708510Initializing all the Stores at 1731341708511 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341708512 (+1 ms)Cleaning up temporary data from old regions at 1731341708518 (+6 ms)Running coprocessor post-open hooks at 1731341708521 (+3 ms)Region opened successfully at 1731341708521 2024-11-11T16:15:08,523 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba., pid=59, masterSystemTime=1731341708503 2024-11-11T16:15:08,525 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,525 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:08,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, openSeqNum=5, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:08,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,33497,1731341698716 because future has completed 2024-11-11T16:15:08,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-11T16:15:08,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 60aabcb1be5df802409ef5ef8a1eccba, server=16b413a53992,33497,1731341698716 in 185 msec 2024-11-11T16:15:08,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE in 541 msec 2024-11-11T16:15:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-11T16:15:08,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:08,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-11T16:15:08,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=639, MaxFileDescriptor=1048576, SystemLoadAverage=705, ProcessCount=11, AvailableMemoryMB=1437 2024-11-11T16:15:08,638 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-11T16:15:08,639 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-11T16:15:08,639 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 16b413a53992,36245,1731341698838 2024-11-11T16:15:08,640 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 16b413a53992,36245,1731341698838: Simulated kill ***** 2024-11-11T16:15:08,640 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-11T16:15:08,642 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-11T16:15:08,643 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-11T16:15:08,646 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-11T16:15:08,648 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-11T16:15:08,659 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 357407360 }, "NonHeapMemoryUsage": { "committed": 155385856, "init": 7667712, "max": -1, "used": 152967240 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34366, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 101, "ProcessCallTime_min": 0, "ProcessCallTime_max": 47, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 11, "ProcessCallTime_median": 23, "ProcessCallTime_75th_percentile": 35, "ProcessCallTime_90th_percentile": 42, "ProcessCallTime_95th_percentile": 44, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 101, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 101, "QueueCallTime_min": 0, "QueueCallTime_max": 32, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 8, "QueueCallTime_median": 16, "QueueCallTime_75th_percentile": 24, "QueueCallTime_90th_percentile": 28, "QueueCallTime_95th_percentile": 30, "QueueCallTime_98th_percentile": 31, "QueueCallTime_99th_percentile": 31, "QueueCallTime_99.9th_percentile": 31, "QueueCallTime_TimeRangeCount_0-1": 101, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 101, "TotalCallTime_min": 0, "TotalCallTime_max": 47, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 11, "TotalCallTime_median": 23, "TotalCallTime_75th_percentile": 35, "TotalCallTime_90th_percentile": 42, "TotalCallTime_95th_percentile": 44, "TotalCallTime_98th_percentile": 46, "TotalCallTime_99th_percentile": 46, "TotalCallTime_99.9th_percentile": 46, "TotalCallTime_TimeRangeCount_0-1": 101, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 101, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 676, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 101, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 101, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 101, "sentBytes": 54732 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54959", "tag.serverName": "16b413a53992,44579,1731341698497", "tag.clusterId": "77267fe1-1280-4bf0-aa3a-d386e646f2df", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731341698497, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-11T16:15:08,665 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35895 {}] master.MasterRpcServices(700): 16b413a53992,36245,1731341698838 reported a fatal error: ***** ABORTING region server 16b413a53992,36245,1731341698838: Simulated kill ***** 2024-11-11T16:15:08,675 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b413a53992,36245,1731341698838' ***** 2024-11-11T16:15:08,676 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-11T16:15:08,676 INFO [RS:2;16b413a53992:36245 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T16:15:08,676 INFO [RS:2;16b413a53992:36245 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-11T16:15:08,676 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T16:15:08,676 INFO [RS:2;16b413a53992:36245 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-11T16:15:08,676 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:08,676 INFO [RS:2;16b413a53992:36245 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:15:08,677 INFO [RS:2;16b413a53992:36245 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;16b413a53992:36245. 2024-11-11T16:15:08,677 DEBUG [RS:2;16b413a53992:36245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:08,682 DEBUG [RS:2;16b413a53992:36245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:08,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741836_1012 (size=33749) 2024-11-11T16:15:08,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741836_1012 (size=33749) 2024-11-11T16:15:08,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741836_1012 (size=33749) 2024-11-11T16:15:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741835_1011 (size=2774) 2024-11-11T16:15:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741835_1011 (size=2774) 2024-11-11T16:15:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741835_1011 (size=2774) 2024-11-11T16:15:08,696 DEBUG [RS:2;16b413a53992:36245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:08,696 INFO [RS:2;16b413a53992:36245 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:08,696 INFO [RS:2;16b413a53992:36245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36245 2024-11-11T16:15:08,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:15:08,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b413a53992,36245,1731341698838 2024-11-11T16:15:08,713 INFO [RS:2;16b413a53992:36245 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T16:15:08,714 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b413a53992,36245,1731341698838] 2024-11-11T16:15:08,717 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b413a53992,36245,1731341698838 already deleted, retry=false 2024-11-11T16:15:08,717 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 16b413a53992,36245,1731341698838 on 16b413a53992,35895,1731341697364 2024-11-11T16:15:08,723 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:08,728 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 16b413a53992,36245,1731341698838 (carryingMeta=true) 16b413a53992,36245,1731341698838/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6964cd1d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-11T16:15:08,729 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:08,732 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true, isMeta: true 2024-11-11T16:15:08,733 INFO [regionserver/16b413a53992:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:08,739 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting 2024-11-11T16:15:08,741 INFO [PEWorker-1 {}] master.SplitWALManager(105): 16b413a53992,36245,1731341698838 WAL count=1, meta=true 2024-11-11T16:15:08,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta}] 2024-11-11T16:15:08,752 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:08,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:08,778 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T16:15:08,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:15:08,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36245-0x1002f9fc70b0003, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:15:08,818 INFO [RS:2;16b413a53992:36245 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T16:15:08,818 INFO [RS:2;16b413a53992:36245 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b413a53992,36245,1731341698838; zookeeper connection closed. 2024-11-11T16:15:08,819 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12d60431 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12d60431 2024-11-11T16:15:08,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33497 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-11T16:15:08,941 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, size=33.0 K (33749bytes) 2024-11-11T16:15:08,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-11T16:15:08,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-11T16:15:08,942 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta 2024-11-11T16:15:08,947 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta after 4ms 2024-11-11T16:15:08,957 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta took 17ms 2024-11-11T16:15:08,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-11T16:15:08,983 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T16:15:08,983 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T16:15:08,996 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp 2024-11-11T16:15:09,003 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta so closing down 2024-11-11T16:15:09,003 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-11T16:15:09,011 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp 2024-11-11T16:15:09,015 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-11T16:15:09,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741864_1040 (size=33508) 2024-11-11T16:15:09,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741864_1040 (size=33508) 2024-11-11T16:15:09,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741864_1040 (size=33508) 2024-11-11T16:15:09,053 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-11-11T16:15:09,064 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-11T16:15:09,070 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 102 ms; skipped=1; WAL=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-11T16:15:09,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, journal: Splitting hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, size=33.0 K (33749bytes) at 1731341708941Finishing writing output for hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta so closing down at 1731341709003 (+62 ms)Creating recovered edits writer path=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp at 1731341709011 (+8 ms)3 split writer threads finished at 1731341709015 (+4 ms)Closed recovered edits writer path=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1731341709053 (+38 ms)Rename recovered edits hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta.temp to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731341709064 (+11 ms)Processed 62 edits across 1 Regions in 102 ms; skipped=1; WAL=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731341709070 (+6 ms) 2024-11-11T16:15:09,071 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta 2024-11-11T16:15:09,073 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-11T16:15:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-11T16:15:09,091 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs 2024-11-11T16:15:09,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-11T16:15:09,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 in 338 msec 2024-11-11T16:15:09,106 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:09,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T16:15:09,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 in 363 msec 2024-11-11T16:15:09,120 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting dir is empty, no logs to split. 2024-11-11T16:15:09,120 INFO [PEWorker-1 {}] master.SplitWALManager(105): 16b413a53992,36245,1731341698838 WAL count=0, meta=true 2024-11-11T16:15:09,120 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 16b413a53992,36245,1731341698838 WAL splitting is done? wals=0, meta=true 2024-11-11T16:15:09,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T16:15:09,125 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:09,126 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:09,276 DEBUG [16b413a53992:35895 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(204): Hosts are {16b413a53992=0} racks are {/default-rack=0} 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T16:15:09,277 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T16:15:09,277 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T16:15:09,277 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T16:15:09,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:09,279 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPENING 2024-11-11T16:15:09,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:09,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:09,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:09,294 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:09,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:09,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:09,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:09,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:09,453 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T16:15:09,453 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:15:09,454 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T16:15:09,458 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C33497%2C1731341698716.meta, suffix=.meta, logDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716, archiveDir=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs, maxLogs=32 2024-11-11T16:15:09,478 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716/16b413a53992%2C33497%2C1731341698716.meta.1731341709459.meta, exclude list is [], retry=0 2024-11-11T16:15:09,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45315,DS-bdf836a0-cb30-404e-a2f7-e03c864f21ce,DISK] 2024-11-11T16:15:09,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37169,DS-fb1a0eed-38c2-4fcc-bc4d-c2c3c6f5eb8c,DISK] 2024-11-11T16:15:09,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40759,DS-8b7d60eb-133c-4577-9fbf-d21dff0f3db2,DISK] 2024-11-11T16:15:09,489 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,33497,1731341698716/16b413a53992%2C33497%2C1731341698716.meta.1731341709459.meta 2024-11-11T16:15:09,497 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41647:41647),(127.0.0.1/127.0.0.1:33177:33177),(127.0.0.1/127.0.0.1:33195:33195)] 2024-11-11T16:15:09,497 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:15:09,497 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T16:15:09,497 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T16:15:09,498 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T16:15:09,498 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T16:15:09,498 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:09,498 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T16:15:09,498 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T16:15:09,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T16:15:09,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T16:15:09,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:09,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:09,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T16:15:09,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T16:15:09,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:09,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:09,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T16:15:09,522 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T16:15:09,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:09,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:09,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T16:15:09,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T16:15:09,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:09,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:15:09,526 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T16:15:09,527 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:09,530 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740 2024-11-11T16:15:09,531 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-11T16:15:09,571 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-11T16:15:09,576 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-11T16:15:09,674 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/998ffa24fbdb4712bc96602ff9329bdb is 138, key is test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2./info:regioninfo/1731341705191/Put/seqid=0 2024-11-11T16:15:09,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741866_1042 (size=36371) 2024-11-11T16:15:09,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741866_1042 (size=36371) 2024-11-11T16:15:09,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741866_1042 (size=36371) 2024-11-11T16:15:09,717 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/998ffa24fbdb4712bc96602ff9329bdb 2024-11-11T16:15:09,815 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/ns/dec6ca66a281497e91b3f13096d1eb2d is 43, key is default/ns:d/1731341702218/Put/seqid=0 2024-11-11T16:15:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741867_1043 (size=5153) 2024-11-11T16:15:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741867_1043 (size=5153) 2024-11-11T16:15:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741867_1043 (size=5153) 2024-11-11T16:15:09,890 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/ns/dec6ca66a281497e91b3f13096d1eb2d 2024-11-11T16:15:09,924 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/table/0d869dd6803146879c43c57462d39f3f is 40, key is test/table:state/1731341705367/Put/seqid=0 2024-11-11T16:15:09,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741868_1044 (size=5165) 2024-11-11T16:15:09,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741868_1044 (size=5165) 2024-11-11T16:15:09,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741868_1044 (size=5165) 2024-11-11T16:15:09,946 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/table/0d869dd6803146879c43c57462d39f3f 2024-11-11T16:15:09,966 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/998ffa24fbdb4712bc96602ff9329bdb as hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/info/998ffa24fbdb4712bc96602ff9329bdb 2024-11-11T16:15:09,983 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/info/998ffa24fbdb4712bc96602ff9329bdb, entries=273, sequenceid=64, filesize=35.5 K 2024-11-11T16:15:09,987 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/ns/dec6ca66a281497e91b3f13096d1eb2d as hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/ns/dec6ca66a281497e91b3f13096d1eb2d 2024-11-11T16:15:09,997 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/ns/dec6ca66a281497e91b3f13096d1eb2d, entries=2, sequenceid=64, filesize=5.0 K 2024-11-11T16:15:09,998 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/table/0d869dd6803146879c43c57462d39f3f as hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/table/0d869dd6803146879c43c57462d39f3f 2024-11-11T16:15:10,007 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/table/0d869dd6803146879c43c57462d39f3f, entries=2, sequenceid=64, filesize=5.0 K 2024-11-11T16:15:10,013 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 435ms, sequenceid=64, compaction requested=false; wal=null 2024-11-11T16:15:10,015 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-11T16:15:10,017 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T16:15:10,017 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T16:15:10,019 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T16:15:10,021 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T16:15:10,024 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-11T16:15:10,025 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69388445, jitterRate=0.033968403935432434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T16:15:10,026 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T16:15:10,026 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731341709499Writing region info on filesystem at 1731341709499Initializing all the Stores at 1731341709501 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341709501Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341709503 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341709503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341709503Obtaining lock to block concurrent updates at 1731341709576 (+73 ms)Preparing flush snapshotting stores in 1588230740 at 1731341709576Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731341709579 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731341709580 (+1 ms)Flushing 1588230740/info: creating writer at 1731341709582 (+2 ms)Flushing 1588230740/info: appending metadata at 1731341709658 (+76 ms)Flushing 1588230740/info: closing flushed file at 1731341709662 (+4 ms)Flushing 1588230740/ns: creating writer at 1731341709783 (+121 ms)Flushing 1588230740/ns: appending metadata at 1731341709813 (+30 ms)Flushing 1588230740/ns: closing flushed file at 1731341709813Flushing 1588230740/table: creating writer at 1731341709898 (+85 ms)Flushing 1588230740/table: appending metadata at 1731341709923 (+25 ms)Flushing 1588230740/table: closing flushed file at 1731341709923Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ee6da7: reopening flushed file at 1731341709963 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28604a7b: reopening flushed file at 1731341709986 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3addccdc: reopening flushed file at 1731341709997 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 435ms, sequenceid=64, compaction requested=false; wal=null at 1731341710013 (+16 ms)Cleaning up temporary data from old regions at 1731341710017 (+4 ms)Running coprocessor post-open hooks at 1731341710026 (+9 ms)Region opened successfully at 1731341710026 2024-11-11T16:15:10,028 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731341709446 2024-11-11T16:15:10,031 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T16:15:10,032 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T16:15:10,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,035 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,038 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T16:15:10,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 in 744 msec 2024-11-11T16:15:10,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-11T16:15:10,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 919 msec 2024-11-11T16:15:10,045 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 16b413a53992,36245,1731341698838 had 9 regions 2024-11-11T16:15:10,046 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true, isMeta: false 2024-11-11T16:15:10,049 INFO [PEWorker-1 {}] master.SplitWALManager(105): 16b413a53992,36245,1731341698838 WAL count=1, meta=false 2024-11-11T16:15:10,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116}] 2024-11-11T16:15:10,052 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:10,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44579 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-11T16:15:10,225 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116, size=2.7 K (2774bytes) 2024-11-11T16:15:10,225 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 2024-11-11T16:15:10,226 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 after 0ms 2024-11-11T16:15:10,230 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 took 5ms 2024-11-11T16:15:10,233 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 60aabcb1be5df802409ef5ef8a1eccba: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,235 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 7ee84052923dbaf7f5c9581236276c4c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,237 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f9f064481e2ac446aeb716014e0cdf6b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,238 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9984848611dae4de6b309aeff78ed621: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,240 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 97c9a93ca402a988eed0542bbe16f17a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,241 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 58c93273fb8e84a5a8b730fc9fafb6f7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,243 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for aebcd9c83edd713d66ccd5d8f1a5150d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,245 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b7e69cf109199b3c3d754201ea9818cd: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,246 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5317ee89294e310637e2785c5a255b6b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T16:15:10,246 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 so closing down 2024-11-11T16:15:10,246 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-11T16:15:10,246 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-11T16:15:10,246 INFO [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-11T16:15:10,246 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116, journal: Splitting hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116, size=2.7 K (2774bytes) at 1731341710225Finishing writing output for hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 so closing down at 1731341710246 (+21 ms)3 split writer threads finished at 1731341710246Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731341710246 2024-11-11T16:15:10,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 2024-11-11T16:15:10,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-11T16:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35895 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-11T16:15:10,252 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting/16b413a53992%2C36245%2C1731341698838.1731341701116 to hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs 2024-11-11T16:15:10,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-11T16:15:10,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 in 199 msec 2024-11-11T16:15:10,256 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-11T16:15:10,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 in 208 msec 2024-11-11T16:15:10,262 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/WALs/16b413a53992,36245,1731341698838-splitting dir is empty, no logs to split. 2024-11-11T16:15:10,262 INFO [PEWorker-1 {}] master.SplitWALManager(105): 16b413a53992,36245,1731341698838 WAL count=0, meta=false 2024-11-11T16:15:10,262 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 16b413a53992,36245,1731341698838 WAL splitting is done? wals=0, meta=false 2024-11-11T16:15:10,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN}] 2024-11-11T16:15:10,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,268 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,269 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,270 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,271 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,272 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,273 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,275 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,275 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,276 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,278 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,279 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,280 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(204): Hosts are {16b413a53992=0} racks are {/default-rack=0} 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T16:15:10,418 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T16:15:10,418 INFO [16b413a53992:35895 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T16:15:10,418 DEBUG [16b413a53992:35895 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T16:15:10,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,420 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 16b413a53992:36245 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:10,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, regionLocation=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9984848611dae4de6b309aeff78ed621, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-11T16:15:10,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 from cache 2024-11-11T16:15:10,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,423 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:10,423 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1] 2024-11-11T16:15:10,423 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:10,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 16b413a53992:36245 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, regionLocation=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:10,425 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 16b413a53992:36245 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:10,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=16b413a53992,36245,1731341698838, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=16b413a53992,33497,1731341698716, seqNum=-1, error=java.net.ConnectException: Call to address=16b413a53992:36245 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 16b413a53992:36245 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 16b413a53992/172.17.0.2:36245 2024-11-11T16:15:10,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN because future has completed 2024-11-11T16:15:10,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:10,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN because future has completed 2024-11-11T16:15:10,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:10,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN because future has completed 2024-11-11T16:15:10,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:10,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN because future has completed 2024-11-11T16:15:10,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:10,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN because future has completed 2024-11-11T16:15:10,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN because future has completed 2024-11-11T16:15:10,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN because future has completed 2024-11-11T16:15:10,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN because future has completed 2024-11-11T16:15:10,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:10,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497}] 2024-11-11T16:15:10,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:10,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,33497,1731341698716}] 2024-11-11T16:15:10,586 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:10,586 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 9984848611dae4de6b309aeff78ed621, NAME => 'test,www,1731341702648.9984848611dae4de6b309aeff78ed621.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-11T16:15:10,587 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,587 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,www,1731341702648.9984848611dae4de6b309aeff78ed621.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:10,587 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,587 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,588 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,589 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9984848611dae4de6b309aeff78ed621 columnFamilyName family 2024-11-11T16:15:10,590 DEBUG [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,590 INFO [StoreOpener-9984848611dae4de6b309aeff78ed621-1 {}] regionserver.HStore(327): Store=9984848611dae4de6b309aeff78ed621/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:10,590 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,591 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,592 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,593 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,593 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,595 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,597 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 9984848611dae4de6b309aeff78ed621; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67376141, jitterRate=0.00398273766040802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:10,597 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:10,597 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 9984848611dae4de6b309aeff78ed621: Running coprocessor pre-open hook at 1731341710587Writing region info on filesystem at 1731341710587Initializing all the Stores at 1731341710588 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341710588Cleaning up temporary data from old regions at 1731341710593 (+5 ms)Running coprocessor post-open hooks at 1731341710597 (+4 ms)Region opened successfully at 1731341710597 2024-11-11T16:15:10,598 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731341702648.9984848611dae4de6b309aeff78ed621., pid=80, masterSystemTime=1731341710581 2024-11-11T16:15:10,600 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:10,601 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:10,601 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:10,601 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => b7e69cf109199b3c3d754201ea9818cd, NAME => 'test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-11T16:15:10,601 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,601 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:10,602 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,602 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,603 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,604 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-11T16:15:10,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,604 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e69cf109199b3c3d754201ea9818cd columnFamilyName family 2024-11-11T16:15:10,605 DEBUG [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,605 INFO [StoreOpener-b7e69cf109199b3c3d754201ea9818cd-1 {}] regionserver.HStore(327): Store=b7e69cf109199b3c3d754201ea9818cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:10,605 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,604 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,606 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,608 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,609 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,609 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,611 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,613 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened b7e69cf109199b3c3d754201ea9818cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60266657, jitterRate=-0.10195682942867279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:10,613 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:10,613 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for b7e69cf109199b3c3d754201ea9818cd: Running coprocessor pre-open hook at 1731341710602Writing region info on filesystem at 1731341710602Initializing all the Stores at 1731341710603 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341710603Cleaning up temporary data from old regions at 1731341710609 (+6 ms)Running coprocessor post-open hooks at 1731341710613 (+4 ms)Region opened successfully at 1731341710613 2024-11-11T16:15:10,614 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd., pid=75, masterSystemTime=1731341710581 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:10,617 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:10,617 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => aebcd9c83edd713d66ccd5d8f1a5150d, NAME => 'test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,617 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,616 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-11T16:15:10,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,618 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,619 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,620 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebcd9c83edd713d66ccd5d8f1a5150d columnFamilyName family 2024-11-11T16:15:10,620 DEBUG [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,621 INFO [StoreOpener-aebcd9c83edd713d66ccd5d8f1a5150d-1 {}] regionserver.HStore(327): Store=aebcd9c83edd713d66ccd5d8f1a5150d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,621 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,623 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,623 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,623 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,625 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,626 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened aebcd9c83edd713d66ccd5d8f1a5150d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70580005, jitterRate=0.05172403156757355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:10,626 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:10,626 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for aebcd9c83edd713d66ccd5d8f1a5150d: Running coprocessor pre-open hook at 1731341710617Writing region info on filesystem at 1731341710617Initializing all the Stores at 1731341710618 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341710618Cleaning up temporary data from old regions at 1731341710623 (+5 ms)Running coprocessor post-open hooks at 1731341710626 (+3 ms)Region opened successfully at 1731341710626 2024-11-11T16:15:10,627 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d., pid=76, masterSystemTime=1731341710581 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:10,630 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:10,630 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c93273fb8e84a5a8b730fc9fafb6f7, NAME => 'test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,630 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,632 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,633 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c93273fb8e84a5a8b730fc9fafb6f7 columnFamilyName family 2024-11-11T16:15:10,633 DEBUG [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,634 INFO [StoreOpener-58c93273fb8e84a5a8b730fc9fafb6f7-1 {}] regionserver.HStore(327): Store=58c93273fb8e84a5a8b730fc9fafb6f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:10,634 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,635 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,637 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,637 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,637 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,640 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,641 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 58c93273fb8e84a5a8b730fc9fafb6f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75386270, jitterRate=0.12334296107292175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:10,641 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:10,641 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 58c93273fb8e84a5a8b730fc9fafb6f7: Running coprocessor pre-open hook at 1731341710630Writing region info on filesystem at 1731341710630Initializing all the Stores at 1731341710632 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341710632Cleaning up temporary data from old regions at 1731341710637 (+5 ms)Running coprocessor post-open hooks at 1731341710641 (+4 ms)Region opened successfully at 1731341710641 2024-11-11T16:15:10,642 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7., pid=79, masterSystemTime=1731341710581 2024-11-11T16:15:10,644 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:10,645 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:10,686 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,686 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,687 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,687 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,687 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,688 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,688 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:10,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:10,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ee84052923dbaf7f5c9581236276c4c, NAME => 'test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-11T16:15:10,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:10,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,691 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,692 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ee84052923dbaf7f5c9581236276c4c columnFamilyName family 2024-11-11T16:15:10,692 DEBUG [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,693 INFO [StoreOpener-7ee84052923dbaf7f5c9581236276c4c-1 {}] regionserver.HStore(327): Store=7ee84052923dbaf7f5c9581236276c4c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:10,693 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,694 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,695 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,695 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,695 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,701 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,702 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 7ee84052923dbaf7f5c9581236276c4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63007601, jitterRate=-0.06111358106136322}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:10,702 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:10,702 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 7ee84052923dbaf7f5c9581236276c4c: Running coprocessor pre-open hook at 1731341710689Writing region info on filesystem at 1731341710689Initializing all the Stores at 1731341710691 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341710691Cleaning up temporary data from old regions at 1731341710696 (+5 ms)Running coprocessor post-open hooks at 1731341710702 (+6 ms)Region opened successfully at 1731341710702 2024-11-11T16:15:10,706 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c., pid=77, masterSystemTime=1731341710684 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,713 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5317ee89294e310637e2785c5a255b6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 26 msec 2024-11-11T16:15:10,719 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "16b413a53992" port: 33497 start_code: 1731341698716 } transition { transition_code: OPENED region_info { region_id: 1731341702648 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731341699012 }; retry (#0) after 1007ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-11T16:15:10,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:10,729 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:10,729 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,735 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,736 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,736 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,753 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:10,753 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:10,753 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:10,753 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:10,753 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:10,754 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:10,754 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:10,754 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:10,760 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341740760 2024-11-11T16:15:10,761 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:10,762 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,765 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:10,774 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,774 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,775 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:10,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,825 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:10,826 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341770826 2024-11-11T16:15:10,826 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341830826 2024-11-11T16:15:10,826 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 65 msec 2024-11-11T16:15:10,826 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,826 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,826 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,827 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,827 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,827 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,828 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,846 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,846 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97c9a93ca402a988eed0542bbe16f17a, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 18 msec 2024-11-11T16:15:10,850 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:10,853 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:10,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:10,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:10,877 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341740876 2024-11-11T16:15:10,877 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:10,877 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,879 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:10,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,888 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:10,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,894 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:10,895 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341770895 2024-11-11T16:15:10,895 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341830895 2024-11-11T16:15:10,895 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 18 msec 2024-11-11T16:15:10,895 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,895 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,895 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,897 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,897 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,897 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,897 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T16:15:10,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-11T16:15:10,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:10,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:10,912 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:10,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:10,918 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341740918 2024-11-11T16:15:10,918 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:10,919 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:10,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,922 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,922 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:10,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,932 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:10,932 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341770932 2024-11-11T16:15:10,932 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341830932 2024-11-11T16:15:10,932 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-11T16:15:10,932 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,932 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,933 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,933 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,933 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,933 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,933 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,938 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,939 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,939 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,939 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,945 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c93273fb8e84a5a8b730fc9fafb6f7, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-11T16:15:10,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:10,946 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:10,946 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,949 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:10,949 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:10,950 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341740950 2024-11-11T16:15:10,950 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:10,951 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,952 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:10,953 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,953 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,954 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,958 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:10,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,964 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:10,964 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341770964 2024-11-11T16:15:10,964 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341830964 2024-11-11T16:15:10,964 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-11T16:15:10,964 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,964 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,965 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,965 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,965 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,965 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,966 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,966 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,975 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,976 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,976 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9f064481e2ac446aeb716014e0cdf6b, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,977 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-11T16:15:10,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:10,978 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:10,978 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:10,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:10,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:10,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:10,982 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341740982 2024-11-11T16:15:10,982 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:10,982 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,984 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:10,987 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:10,987 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,987 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:10,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:10,991 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:10,991 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341770991 2024-11-11T16:15:10,991 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341830991 2024-11-11T16:15:10,991 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:10,991 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:10,991 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:10,992 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:10,992 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:10,992 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:10,992 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:10,992 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:10,993 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:10,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:10,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:10,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee84052923dbaf7f5c9581236276c4c, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:10,999 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T16:15:11,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,000 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,000 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:11,004 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:11,004 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741004 2024-11-11T16:15:11,004 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,004 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,006 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,007 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,007 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,007 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,013 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,013 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771013 2024-11-11T16:15:11,013 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831013 2024-11-11T16:15:11,014 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:11,014 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,014 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,014 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,015 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,015 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,015 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-11T16:15:11,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,025 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,025 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,025 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebcd9c83edd713d66ccd5d8f1a5150d, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-11T16:15:11,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,027 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,027 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,031 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,031 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,032 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,033 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:11,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:11,044 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741044 2024-11-11T16:15:11,045 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,045 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,046 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,048 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,057 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771057 2024-11-11T16:15:11,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831057 2024-11-11T16:15:11,057 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-11T16:15:11,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,057 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,057 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,066 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,066 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,067 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,067 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,070 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,073 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,073 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,074 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,074 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,074 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,074 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,074 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,074 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,074 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e69cf109199b3c3d754201ea9818cd, server=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T16:15:11,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,075 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,075 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,077 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:11,078 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741078 2024-11-11T16:15:11,078 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,079 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,081 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,088 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,088 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771088 2024-11-11T16:15:11,088 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831088 2024-11-11T16:15:11,088 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:11,088 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,088 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,088 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,089 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,089 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,089 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,089 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,092 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN 2024-11-11T16:15:11,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T16:15:11,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,097 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,097 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:11,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7e69cf109199b3c3d754201ea9818cd, ASSIGN to state=OFFLINE, location=null, table=test, region=b7e69cf109199b3c3d754201ea9818cd to restore RIT 2024-11-11T16:15:11,101 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741101 2024-11-11T16:15:11,101 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,101 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,102 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,103 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,103 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,104 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,112 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771112 2024-11-11T16:15:11,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831112 2024-11-11T16:15:11,112 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-11T16:15:11,113 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,113 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,113 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,113 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,113 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,113 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,114 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN 2024-11-11T16:15:11,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T16:15:11,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,121 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d, ASSIGN to state=OFFLINE, location=null, table=test, region=aebcd9c83edd713d66ccd5d8f1a5150d to restore RIT 2024-11-11T16:15:11,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741125 2024-11-11T16:15:11,125 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,126 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,128 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,129 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,136 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771136 2024-11-11T16:15:11,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831136 2024-11-11T16:15:11,136 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-11T16:15:11,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,136 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,136 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,137 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,137 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,145 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,145 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,145 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,145 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,145 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,145 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN 2024-11-11T16:15:11,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T16:15:11,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,146 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,146 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,149 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,149 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,149 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,149 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,149 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,150 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97c9a93ca402a988eed0542bbe16f17a, ASSIGN to state=OFFLINE, location=null, table=test, region=97c9a93ca402a988eed0542bbe16f17a to restore RIT 2024-11-11T16:15:11,150 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741150 2024-11-11T16:15:11,150 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,150 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,153 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,160 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771160 2024-11-11T16:15:11,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831161 2024-11-11T16:15:11,161 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-11T16:15:11,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,161 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,161 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,161 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,162 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN 2024-11-11T16:15:11,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T16:15:11,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,185 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,186 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9f064481e2ac446aeb716014e0cdf6b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9f064481e2ac446aeb716014e0cdf6b to restore RIT 2024-11-11T16:15:11,205 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741205 2024-11-11T16:15:11,205 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,205 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,207 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,209 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,215 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771215 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831215 2024-11-11T16:15:11,215 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,215 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,215 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,215 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN 2024-11-11T16:15:11,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-11T16:15:11,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,231 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7ee84052923dbaf7f5c9581236276c4c, ASSIGN to state=OFFLINE, location=null, table=test, region=7ee84052923dbaf7f5c9581236276c4c to restore RIT 2024-11-11T16:15:11,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,252 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741251 2024-11-11T16:15:11,252 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,252 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,261 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771261 2024-11-11T16:15:11,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831261 2024-11-11T16:15:11,261 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:11,261 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,261 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,261 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,261 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,262 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,262 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,262 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T16:15:11,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,268 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,269 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,269 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN 2024-11-11T16:15:11,269 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T16:15:11,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,270 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,271 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,275 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,280 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9984848611dae4de6b309aeff78ed621, ASSIGN to state=OFFLINE, location=null, table=test, region=9984848611dae4de6b309aeff78ed621 to restore RIT 2024-11-11T16:15:11,284 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741284 2024-11-11T16:15:11,285 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,285 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,287 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,287 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,288 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,294 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,294 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771294 2024-11-11T16:15:11,294 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831294 2024-11-11T16:15:11,294 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:11,294 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,294 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,294 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,295 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,295 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,295 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,301 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,301 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN 2024-11-11T16:15:11,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T16:15:11,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5317ee89294e310637e2785c5a255b6b, ASSIGN to state=OFFLINE, location=null, table=test, region=5317ee89294e310637e2785c5a255b6b to restore RIT 2024-11-11T16:15:11,328 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741328 2024-11-11T16:15:11,329 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,330 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,332 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,337 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,338 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,338 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,338 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,338 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771338 2024-11-11T16:15:11,338 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831338 2024-11-11T16:15:11,338 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-11T16:15:11,339 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,340 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,588 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,588 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,589 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,594 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,594 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN 2024-11-11T16:15:11,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T16:15:11,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,595 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,595 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7, ASSIGN to state=OFFLINE, location=null, table=test, region=58c93273fb8e84a5a8b730fc9fafb6f7 to restore RIT 2024-11-11T16:15:11,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741598 2024-11-11T16:15:11,598 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,598 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,600 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,601 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,604 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,604 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771604 2024-11-11T16:15:11,604 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831604 2024-11-11T16:15:11,604 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-11T16:15:11,604 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,605 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,605 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,605 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,605 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,605 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,606 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,609 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T16:15:11,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,609 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741612 2024-11-11T16:15:11,612 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,612 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,615 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,615 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,615 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,619 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771619 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831619 2024-11-11T16:15:11,619 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,619 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,619 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,619 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,619 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,623 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.1731341701116, worker=16b413a53992,44579,1731341698497 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T16:15:11,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,633 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,633 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,638 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741638 2024-11-11T16:15:11,638 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,638 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,641 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,641 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,641 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,645 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,645 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771645 2024-11-11T16:15:11,645 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831645 2024-11-11T16:15:11,645 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-11T16:15:11,645 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,645 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,645 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,646 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,646 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,646 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,647 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T16:15:11,649 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T16:15:11,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,652 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,652 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741656 2024-11-11T16:15:11,656 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,656 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,659 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,663 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771663 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831663 2024-11-11T16:15:11,663 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,663 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,663 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,663 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,663 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T16:15:11,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,667 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,667 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741672 2024-11-11T16:15:11,672 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,672 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,674 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,679 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771679 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831679 2024-11-11T16:15:11,679 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,679 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,679 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,679 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,679 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,680 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,683 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,684 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,687 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741687 2024-11-11T16:15:11,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,687 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,689 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,690 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,690 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,690 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,695 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771695 2024-11-11T16:15:11,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831695 2024-11-11T16:15:11,695 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-11T16:15:11,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,696 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,696 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,696 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,696 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,699 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,699 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T16:15:11,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,700 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 16b413a53992%2C36245%2C1731341698838.meta.1731341701817.meta, worker=16b413a53992,33497,1731341698716 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T16:15:11,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T16:15:11,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,701 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,702 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741704 2024-11-11T16:15:11,704 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,704 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,706 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,710 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,710 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771710 2024-11-11T16:15:11,710 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831710 2024-11-11T16:15:11,710 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-11T16:15:11,710 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T16:15:11,710 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,710 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,710 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:11,711 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T16:15:11,711 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T16:15:11,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T16:15:11,711 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:15:11,712 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T16:15:11,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T16:15:11,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:15:11,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T16:15:11,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60aabcb1be5df802409ef5ef8a1eccba, REOPEN/MOVE 2024-11-11T16:15:11,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true 2024-11-11T16:15:11,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T16:15:11,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T16:15:11,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=65 2024-11-11T16:15:11,714 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=16b413a53992,33497,1731341698716, table=hbase:meta, region=1588230740 2024-11-11T16:15:11,715 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,33497,1731341698716, state=OPEN 2024-11-11T16:15:11,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:15:11,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:15:11,717 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341741717 2024-11-11T16:15:11,717 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:15:11,717 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:11,718 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T16:15:11,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 9984848611dae4de6b309aeff78ed621, server=16b413a53992,44579,1731341698497:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 16b413a53992,36245,1731341698838, splitWal=true, meta=true exec-time=3.0000 sec 2024-11-11T16:15:11,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62d867b8a89d21a5e2538e1e5222da88, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d05687468bd618dc7fa99d7c6b5f3b2, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aebcd9c83edd713d66ccd5d8f1a5150d, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97c9a93ca402a988eed0542bbe16f17a, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f2e670caa2864b9154b87162f54c642, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7e69cf109199b3c3d754201ea9818cd, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=756a2cfebbc98a6396c782a802598ffc, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8845a47073d69ee0a0021d18e08e51ef, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd1d8bee83938f0837b5cdacde7e89c4, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5317ee89294e310637e2785c5a255b6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba7b9cc976155fbb76bf00c2ffd4495, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f42aec7889fcb07b96681cea332b42d, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c93273fb8e84a5a8b730fc9fafb6f7, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df81ce6daca1d892d80d840de0c0b8b, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98ffcdd491bf651a96a2bf2c2b5a9a95, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9f064481e2ac446aeb716014e0cdf6b, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35b42287bbbc0c5919124a8c4b800ea1, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e3cef8c6c3398e017e8dfa5cd0c9c242, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d939495820cb0d73d77301eb7bbcb7fb, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60aabcb1be5df802409ef5ef8a1eccba, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=5 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e762741692724ea54cb4ce74a7e615ae, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c2941f7d018cc329ed386329a479fbf, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9984848611dae4de6b309aeff78ed621, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7fb28854349ec0d596f8f0dca65b13b8, regionState=OPEN, lastHost=16b413a53992,44579,1731341698497, regionLocation=16b413a53992,44579,1731341698497, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e62b8abd36e22d5d589c5bdd191ed105, regionState=OPEN, lastHost=16b413a53992,33497,1731341698716, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ee84052923dbaf7f5c9581236276c4c, regionState=OPENING, lastHost=16b413a53992,36245,1731341698838, regionLocation=16b413a53992,33497,1731341698716, openSeqNum=2 2024-11-11T16:15:11,723 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-11T16:15:11,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341771723 2024-11-11T16:15:11,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341831723 2024-11-11T16:15:11,723 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-11T16:15:11,732 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1308): No matching procedure found for 16b413a53992,33497,1731341698716 transition on state=OPENING, location=16b413a53992,33497,1731341698716, table=test, region=7ee84052923dbaf7f5c9581236276c4c to OPENED 2024-11-11T16:15:11,732 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "16b413a53992" port: 33497 start_code: 1731341698716 } transition { transition_code: OPENED region_info { region_id: 1731341702648 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731341699012 } 2024-11-11T16:15:11,732 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,732 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,733 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,733 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 5317ee89294e310637e2785c5a255b6b, NAME => 'test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-11T16:15:11,733 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,733 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:11,733 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,733 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,735 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,737 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5317ee89294e310637e2785c5a255b6b columnFamilyName family 2024-11-11T16:15:11,737 DEBUG [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:11,737 INFO [StoreOpener-5317ee89294e310637e2785c5a255b6b-1 {}] regionserver.HStore(327): Store=5317ee89294e310637e2785c5a255b6b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:11,737 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,739 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 380), OpenFileDescriptor=618 (was 639), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 705), ProcessCount=11 (was 11), AvailableMemoryMB=1282 (was 1437) 2024-11-11T16:15:11,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T16:15:11,740 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:15:11,740 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:11,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,741 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:11,742 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T16:15:11,742 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=778508352, stopped=false 2024-11-11T16:15:11,742 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,742 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,742 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16b413a53992,35895,1731341697364 2024-11-11T16:15:11,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:11,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:11,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T16:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:11,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:11,746 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,747 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 5317ee89294e310637e2785c5a255b6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74818051, jitterRate=0.11487583816051483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:11,747 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,747 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 5317ee89294e310637e2785c5a255b6b: Running coprocessor pre-open hook at 1731341711733Writing region info on filesystem at 1731341711733Initializing all the Stores at 1731341711734 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341711735 (+1 ms)Cleaning up temporary data from old regions at 1731341711742 (+7 ms)Running coprocessor post-open hooks at 1731341711747 (+5 ms)Region opened successfully at 1731341711747 2024-11-11T16:15:11,748 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b., pid=82, masterSystemTime=1731341710684 2024-11-11T16:15:11,749 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:11,749 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35895 {}] assignment.AssignmentManager(1308): No matching procedure found for 16b413a53992,33497,1731341698716 transition on state=OPENING, location=16b413a53992,33497,1731341698716, table=test, region=5317ee89294e310637e2785c5a255b6b to OPENED 2024-11-11T16:15:11,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,750 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b413a53992,44579,1731341698497' ***** 2024-11-11T16:15:11,750 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T16:15:11,750 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b413a53992,33497,1731341698716' ***** 2024-11-11T16:15:11,750 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T16:15:11,750 INFO [RS:0;16b413a53992:44579 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T16:15:11,750 INFO [RS:1;16b413a53992:33497 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T16:15:11,750 INFO [RS:0;16b413a53992:44579 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T16:15:11,750 INFO [RS:0;16b413a53992:44579 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T16:15:11,750 INFO [RS:1;16b413a53992:33497 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T16:15:11,750 INFO [RS:1;16b413a53992:33497 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T16:15:11,750 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 4f42aec7889fcb07b96681cea332b42d 2024-11-11T16:15:11,750 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 62d867b8a89d21a5e2538e1e5222da88 2024-11-11T16:15:11,750 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T16:15:11,750 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T16:15:11,752 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,752 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,753 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. 2024-11-11T16:15:11,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 97c9a93ca402a988eed0542bbe16f17a, NAME => 'test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-11T16:15:11,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:11,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,754 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:15:11,754 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:15:11,755 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 60aabcb1be5df802409ef5ef8a1eccba 2024-11-11T16:15:11,755 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for aebcd9c83edd713d66ccd5d8f1a5150d 2024-11-11T16:15:11,755 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d05687468bd618dc7fa99d7c6b5f3b2 2024-11-11T16:15:11,755 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 5317ee89294e310637e2785c5a255b6b 2024-11-11T16:15:11,755 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4f42aec7889fcb07b96681cea332b42d, disabling compactions & flushes 2024-11-11T16:15:11,755 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 756a2cfebbc98a6396c782a802598ffc 2024-11-11T16:15:11,755 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 9984848611dae4de6b309aeff78ed621 2024-11-11T16:15:11,755 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 62d867b8a89d21a5e2538e1e5222da88, disabling compactions & flushes 2024-11-11T16:15:11,755 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 58c93273fb8e84a5a8b730fc9fafb6f7 2024-11-11T16:15:11,755 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for d939495820cb0d73d77301eb7bbcb7fb 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f2e670caa2864b9154b87162f54c642 2024-11-11T16:15:11,756 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 7fb28854349ec0d596f8f0dca65b13b8 2024-11-11T16:15:11,756 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ee84052923dbaf7f5c9581236276c4c 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for b7e69cf109199b3c3d754201ea9818cd 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ba7b9cc976155fbb76bf00c2ffd4495 2024-11-11T16:15:11,756 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. after waiting 0 ms 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for cd1d8bee83938f0837b5cdacde7e89c4 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 35b42287bbbc0c5919124a8c4b800ea1 2024-11-11T16:15:11,756 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:11,756 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 98ffcdd491bf651a96a2bf2c2b5a9a95 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for e62b8abd36e22d5d589c5bdd191ed105 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(959): stopping server 16b413a53992,33497,1731341698716 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;16b413a53992:33497. 2024-11-11T16:15:11,756 DEBUG [RS:1;16b413a53992:33497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:11,756 DEBUG [RS:1;16b413a53992:33497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,756 INFO [RS:1;16b413a53992:33497 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T16:15:11,757 INFO [RS:1;16b413a53992:33497 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T16:15:11,757 INFO [RS:1;16b413a53992:33497 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T16:15:11,757 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:15:11,757 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T16:15:11,757 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,755 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:11,760 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:11,760 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. after waiting 0 ms 2024-11-11T16:15:11,760 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:11,757 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 8845a47073d69ee0a0021d18e08e51ef 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c2941f7d018cc329ed386329a479fbf 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(3091): Received CLOSE for 7df81ce6daca1d892d80d840de0c0b8b 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(959): stopping server 16b413a53992,44579,1731341698497 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16b413a53992:44579. 2024-11-11T16:15:11,761 DEBUG [RS:0;16b413a53992:44579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:11,761 DEBUG [RS:0;16b413a53992:44579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,761 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-11T16:15:11,762 DEBUG [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(1351): Waiting on 35b42287bbbc0c5919124a8c4b800ea1, 4c2941f7d018cc329ed386329a479fbf, 58c93273fb8e84a5a8b730fc9fafb6f7, 5f2e670caa2864b9154b87162f54c642, 62d867b8a89d21a5e2538e1e5222da88, 7ba7b9cc976155fbb76bf00c2ffd4495, 7df81ce6daca1d892d80d840de0c0b8b, 7fb28854349ec0d596f8f0dca65b13b8, 8845a47073d69ee0a0021d18e08e51ef, 9984848611dae4de6b309aeff78ed621, aebcd9c83edd713d66ccd5d8f1a5150d, b7e69cf109199b3c3d754201ea9818cd, e3cef8c6c3398e017e8dfa5cd0c9c242 2024-11-11T16:15:11,765 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97c9a93ca402a988eed0542bbe16f17a columnFamilyName family 2024-11-11T16:15:11,765 DEBUG [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:11,766 INFO [StoreOpener-97c9a93ca402a988eed0542bbe16f17a-1 {}] regionserver.HStore(327): Store=97c9a93ca402a988eed0542bbe16f17a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:11,766 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,766 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,768 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-11T16:15:11,769 DEBUG [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4f42aec7889fcb07b96681cea332b42d, 5317ee89294e310637e2785c5a255b6b, 60aabcb1be5df802409ef5ef8a1eccba, 6d05687468bd618dc7fa99d7c6b5f3b2, 756a2cfebbc98a6396c782a802598ffc, 7ee84052923dbaf7f5c9581236276c4c, 97c9a93ca402a988eed0542bbe16f17a, 98ffcdd491bf651a96a2bf2c2b5a9a95, cd1d8bee83938f0837b5cdacde7e89c4, d939495820cb0d73d77301eb7bbcb7fb, e62b8abd36e22d5d589c5bdd191ed105, e762741692724ea54cb4ce74a7e615ae 2024-11-11T16:15:11,769 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T16:15:11,769 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T16:15:11,769 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T16:15:11,769 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T16:15:11,769 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T16:15:11,769 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,769 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-11T16:15:11,770 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,770 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,773 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,775 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/62d867b8a89d21a5e2538e1e5222da88/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,776 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 97c9a93ca402a988eed0542bbe16f17a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63935784, jitterRate=-0.04728257656097412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:11,776 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97c9a93ca402a988eed0542bbe16f17a 2024-11-11T16:15:11,776 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 97c9a93ca402a988eed0542bbe16f17a: Running coprocessor pre-open hook at 1731341711753Writing region info on filesystem at 1731341711753Initializing all the Stores at 1731341711756 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341711756Cleaning up temporary data from old regions at 1731341711770 (+14 ms)Running coprocessor post-open hooks at 1731341711776 (+6 ms)Region opened successfully at 1731341711776 2024-11-11T16:15:11,777 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:11,777 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 62d867b8a89d21a5e2538e1e5222da88: Waiting for close lock at 1731341711755Running coprocessor pre-close hooks at 1731341711755Disabling compacts and flushes for region at 1731341711755Disabling writes for close at 1731341711756 (+1 ms)Writing region close event to WAL at 1731341711762 (+6 ms)Running coprocessor post-close hooks at 1731341711777 (+15 ms)Closed at 1731341711777 2024-11-11T16:15:11,777 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731341702648.62d867b8a89d21a5e2538e1e5222da88. 2024-11-11T16:15:11,778 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aebcd9c83edd713d66ccd5d8f1a5150d, disabling compactions & flushes 2024-11-11T16:15:11,778 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:11,778 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:11,778 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. after waiting 0 ms 2024-11-11T16:15:11,778 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:11,778 WARN [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a., aborting... org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 16b413a53992,33497,1731341698716 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:11,778 ERROR [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2440): ***** ABORTING region server 16b413a53992,33497,1731341698716: Failed to open region test,ddd,1731341702648.97c9a93ca402a988eed0542bbe16f17a. and can not recover ***** org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 16b413a53992,33497,1731341698716 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:11,778 ERROR [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-11T16:15:11,779 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-11T16:15:11,780 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-11T16:15:11,781 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-11T16:15:11,782 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-11T16:15:11,786 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 510499456 }, "NonHeapMemoryUsage": { "committed": 164560896, "init": 7667712, "max": -1, "used": 161975408 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34366, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 101, "ProcessCallTime_min": 0, "ProcessCallTime_max": 47, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 11, "ProcessCallTime_median": 23, "ProcessCallTime_75th_percentile": 35, "ProcessCallTime_90th_percentile": 42, "ProcessCallTime_95th_percentile": 44, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 101, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 101, "QueueCallTime_min": 0, "QueueCallTime_max": 32, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 8, "QueueCallTime_median": 16, "QueueCallTime_75th_percentile": 24, "QueueCallTime_90th_percentile": 28, "QueueCallTime_95th_percentile": 30, "QueueCallTime_98th_percentile": 31, "QueueCallTime_99th_percentile": 31, "QueueCallTime_99.9th_percentile": 31, "QueueCallTime_TimeRangeCount_0-1": 101, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 101, "TotalCallTime_min": 0, "TotalCallTime_max": 47, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 11, "TotalCallTime_median": 23, "TotalCallTime_75th_percentile": 35, "TotalCallTime_90th_percentile": 42, "TotalCallTime_95th_percentile": 44, "TotalCallTime_98th_percentile": 46, "TotalCallTime_99th_percentile": 46, "TotalCallTime_99.9th_percentile": 46, "TotalCallTime_TimeRangeCount_0-1": 101, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 101, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 676, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 101, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 101, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 101, "sentBytes": 54732 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54959", "tag.serverName": "16b413a53992,44579,1731341698497", "tag.clusterId": "77267fe1-1280-4bf0-aa3a-d386e646f2df", "tag.Context": "regionserver", "tag.Hostname": "16b413a53992", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731341698497, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-11T16:15:11,789 WARN [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=16b413a53992:35895 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=16b413a53992:35895 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-11T16:15:11,798 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. 2024-11-11T16:15:11,798 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => f9f064481e2ac446aeb716014e0cdf6b, NAME => 'test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-11T16:15:11,798 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,798 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:11,798 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,798 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,801 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/aebcd9c83edd713d66ccd5d8f1a5150d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,802 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:11,802 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aebcd9c83edd713d66ccd5d8f1a5150d: Waiting for close lock at 1731341711778Running coprocessor pre-close hooks at 1731341711778Disabling compacts and flushes for region at 1731341711778Disabling writes for close at 1731341711778Writing region close event to WAL at 1731341711784 (+6 ms)Running coprocessor post-close hooks at 1731341711802 (+18 ms)Closed at 1731341711802 2024-11-11T16:15:11,802 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d. 2024-11-11T16:15:11,803 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9984848611dae4de6b309aeff78ed621, disabling compactions & flushes 2024-11-11T16:15:11,803 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:11,803 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:11,803 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731341702648.9984848611dae4de6b309aeff78ed621. after waiting 0 ms 2024-11-11T16:15:11,803 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:11,804 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4f42aec7889fcb07b96681cea332b42d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,807 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/664e482bea414ad89c6e2e721ca0f3af is 138, key is test,ccc,1731341702648.aebcd9c83edd713d66ccd5d8f1a5150d./info:regioninfo/1731341710424/Put/seqid=0 2024-11-11T16:15:11,809 INFO [regionserver/16b413a53992:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:11,809 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:11,809 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4f42aec7889fcb07b96681cea332b42d: Waiting for close lock at 1731341711755Running coprocessor pre-close hooks at 1731341711755Disabling compacts and flushes for region at 1731341711755Disabling writes for close at 1731341711760 (+5 ms)Writing region close event to WAL at 1731341711773 (+13 ms)Running coprocessor post-close hooks at 1731341711809 (+36 ms)Closed at 1731341711809 2024-11-11T16:15:11,809 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731341702648.4f42aec7889fcb07b96681cea332b42d. 2024-11-11T16:15:11,810 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60aabcb1be5df802409ef5ef8a1eccba, disabling compactions & flushes 2024-11-11T16:15:11,810 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:11,810 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:11,810 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. after waiting 0 ms 2024-11-11T16:15:11,810 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:11,813 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,815 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9f064481e2ac446aeb716014e0cdf6b columnFamilyName family 2024-11-11T16:15:11,815 DEBUG [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:11,816 INFO [StoreOpener-f9f064481e2ac446aeb716014e0cdf6b-1 {}] regionserver.HStore(327): Store=f9f064481e2ac446aeb716014e0cdf6b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:11,816 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,818 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,820 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,820 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,820 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,824 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741869_1045 (size=8192) 2024-11-11T16:15:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741869_1045 (size=8192) 2024-11-11T16:15:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741869_1045 (size=8192) 2024-11-11T16:15:11,826 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/9984848611dae4de6b309aeff78ed621/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,826 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/664e482bea414ad89c6e2e721ca0f3af 2024-11-11T16:15:11,827 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9984848611dae4de6b309aeff78ed621: Waiting for close lock at 1731341711803Running coprocessor pre-close hooks at 1731341711803Disabling compacts and flushes for region at 1731341711803Disabling writes for close at 1731341711803Writing region close event to WAL at 1731341711813 (+10 ms)Running coprocessor post-close hooks at 1731341711827 (+14 ms)Closed at 1731341711827 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731341702648.9984848611dae4de6b309aeff78ed621. 2024-11-11T16:15:11,827 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened f9f064481e2ac446aeb716014e0cdf6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66664896, jitterRate=-0.006615638732910156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:15:11,827 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9f064481e2ac446aeb716014e0cdf6b 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58c93273fb8e84a5a8b730fc9fafb6f7, disabling compactions & flushes 2024-11-11T16:15:11,827 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:11,827 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for f9f064481e2ac446aeb716014e0cdf6b: Running coprocessor pre-open hook at 1731341711799Writing region info on filesystem at 1731341711799Initializing all the Stores at 1731341711800 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341711800Cleaning up temporary data from old regions at 1731341711820 (+20 ms)Running coprocessor post-open hooks at 1731341711827 (+7 ms)Region opened successfully at 1731341711827 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. after waiting 0 ms 2024-11-11T16:15:11,827 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:11,831 WARN [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 16b413a53992,33497,1731341698716 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:11,831 INFO [regionserver/16b413a53992:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:11,831 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ppp,1731341702648.f9f064481e2ac446aeb716014e0cdf6b. and can not recover 2024-11-11T16:15:11,839 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/.tmp/info/664e482bea414ad89c6e2e721ca0f3af as hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/info/664e482bea414ad89c6e2e721ca0f3af 2024-11-11T16:15:11,840 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/60aabcb1be5df802409ef5ef8a1eccba/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-11T16:15:11,841 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:11,841 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60aabcb1be5df802409ef5ef8a1eccba: Waiting for close lock at 1731341711810Running coprocessor pre-close hooks at 1731341711810Disabling compacts and flushes for region at 1731341711810Disabling writes for close at 1731341711810Writing region close event to WAL at 1731341711813 (+3 ms)Running coprocessor post-close hooks at 1731341711841 (+28 ms)Closed at 1731341711841 2024-11-11T16:15:11,841 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731341702648.60aabcb1be5df802409ef5ef8a1eccba. 2024-11-11T16:15:11,842 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d05687468bd618dc7fa99d7c6b5f3b2, disabling compactions & flushes 2024-11-11T16:15:11,842 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:11,842 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:11,842 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. after waiting 0 ms 2024-11-11T16:15:11,842 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:11,849 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/58c93273fb8e84a5a8b730fc9fafb6f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,850 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/info/664e482bea414ad89c6e2e721ca0f3af, entries=24, sequenceid=75, filesize=8 K 2024-11-11T16:15:11,851 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/6d05687468bd618dc7fa99d7c6b5f3b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,851 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d05687468bd618dc7fa99d7c6b5f3b2: Waiting for close lock at 1731341711842Running coprocessor pre-close hooks at 1731341711842Disabling compacts and flushes for region at 1731341711842Disabling writes for close at 1731341711842Writing region close event to WAL at 1731341711845 (+3 ms)Running coprocessor post-close hooks at 1731341711851 (+6 ms)Closed at 1731341711851 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731341702648.6d05687468bd618dc7fa99d7c6b5f3b2. 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5317ee89294e310637e2785c5a255b6b, disabling compactions & flushes 2024-11-11T16:15:11,852 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. after waiting 0 ms 2024-11-11T16:15:11,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,852 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=75, compaction requested=false 2024-11-11T16:15:11,853 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:11,853 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58c93273fb8e84a5a8b730fc9fafb6f7: Waiting for close lock at 1731341711827Running coprocessor pre-close hooks at 1731341711827Disabling compacts and flushes for region at 1731341711827Disabling writes for close at 1731341711827Writing region close event to WAL at 1731341711829 (+2 ms)Running coprocessor post-close hooks at 1731341711853 (+24 ms)Closed at 1731341711853 2024-11-11T16:15:11,853 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731341702648.58c93273fb8e84a5a8b730fc9fafb6f7. 2024-11-11T16:15:11,854 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f2e670caa2864b9154b87162f54c642, disabling compactions & flushes 2024-11-11T16:15:11,854 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:11,854 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:11,854 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. after waiting 0 ms 2024-11-11T16:15:11,854 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:11,858 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5317ee89294e310637e2785c5a255b6b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,860 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5317ee89294e310637e2785c5a255b6b: Waiting for close lock at 1731341711852Running coprocessor pre-close hooks at 1731341711852Disabling compacts and flushes for region at 1731341711852Disabling writes for close at 1731341711852Writing region close event to WAL at 1731341711853 (+1 ms)Running coprocessor post-close hooks at 1731341711859 (+6 ms)Closed at 1731341711860 (+1 ms) 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731341702648.5317ee89294e310637e2785c5a255b6b. 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 756a2cfebbc98a6396c782a802598ffc, disabling compactions & flushes 2024-11-11T16:15:11,860 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. after waiting 0 ms 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:11,860 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/5f2e670caa2864b9154b87162f54c642/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,861 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-11T16:15:11,862 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f2e670caa2864b9154b87162f54c642: Waiting for close lock at 1731341711853Running coprocessor pre-close hooks at 1731341711853Disabling compacts and flushes for region at 1731341711853Disabling writes for close at 1731341711854 (+1 ms)Writing region close event to WAL at 1731341711854Running coprocessor post-close hooks at 1731341711861 (+7 ms)Closed at 1731341711861 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731341702648.5f2e670caa2864b9154b87162f54c642. 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7fb28854349ec0d596f8f0dca65b13b8, disabling compactions & flushes 2024-11-11T16:15:11,862 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. after waiting 0 ms 2024-11-11T16:15:11,862 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:11,863 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T16:15:11,863 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T16:15:11,863 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731341711769Running coprocessor pre-close hooks at 1731341711769Disabling compacts and flushes for region at 1731341711769Disabling writes for close at 1731341711769Obtaining lock to block concurrent updates at 1731341711769Preparing flush snapshotting stores in 1588230740 at 1731341711769Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731341711776 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731341711785 (+9 ms)Flushing 1588230740/info: creating writer at 1731341711786 (+1 ms)Flushing 1588230740/info: appending metadata at 1731341711806 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731341711806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@db0d597: reopening flushed file at 1731341711836 (+30 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=75, compaction requested=false at 1731341711852 (+16 ms)Writing region close event to WAL at 1731341711855 (+3 ms)Running coprocessor post-close hooks at 1731341711862 (+7 ms)Closed at 1731341711863 (+1 ms) 2024-11-11T16:15:11,863 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T16:15:11,868 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/756a2cfebbc98a6396c782a802598ffc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,868 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7fb28854349ec0d596f8f0dca65b13b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,869 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7fb28854349ec0d596f8f0dca65b13b8: Waiting for close lock at 1731341711862Running coprocessor pre-close hooks at 1731341711862Disabling compacts and flushes for region at 1731341711862Disabling writes for close at 1731341711862Writing region close event to WAL at 1731341711863 (+1 ms)Running coprocessor post-close hooks at 1731341711869 (+6 ms)Closed at 1731341711869 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731341702648.7fb28854349ec0d596f8f0dca65b13b8. 2024-11-11T16:15:11,869 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7e69cf109199b3c3d754201ea9818cd, disabling compactions & flushes 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 756a2cfebbc98a6396c782a802598ffc: Waiting for close lock at 1731341711860Running coprocessor pre-close hooks at 1731341711860Disabling compacts and flushes for region at 1731341711860Disabling writes for close at 1731341711860Writing region close event to WAL at 1731341711862 (+2 ms)Running coprocessor post-close hooks at 1731341711869 (+7 ms)Closed at 1731341711869 2024-11-11T16:15:11,869 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. after waiting 0 ms 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731341702648.756a2cfebbc98a6396c782a802598ffc. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:11,869 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d939495820cb0d73d77301eb7bbcb7fb, disabling compactions & flushes 2024-11-11T16:15:11,870 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:11,870 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:11,870 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. after waiting 0 ms 2024-11-11T16:15:11,870 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:11,877 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/d939495820cb0d73d77301eb7bbcb7fb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,877 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:11,877 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d939495820cb0d73d77301eb7bbcb7fb: Waiting for close lock at 1731341711869Running coprocessor pre-close hooks at 1731341711869Disabling compacts and flushes for region at 1731341711869Disabling writes for close at 1731341711870 (+1 ms)Writing region close event to WAL at 1731341711871 (+1 ms)Running coprocessor post-close hooks at 1731341711877 (+6 ms)Closed at 1731341711877 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731341702648.d939495820cb0d73d77301eb7bbcb7fb. 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ee84052923dbaf7f5c9581236276c4c, disabling compactions & flushes 2024-11-11T16:15:11,878 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. after waiting 0 ms 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/b7e69cf109199b3c3d754201ea9818cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,878 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,879 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7e69cf109199b3c3d754201ea9818cd: Waiting for close lock at 1731341711869Running coprocessor pre-close hooks at 1731341711869Disabling compacts and flushes for region at 1731341711869Disabling writes for close at 1731341711869Writing region close event to WAL at 1731341711872 (+3 ms)Running coprocessor post-close hooks at 1731341711879 (+7 ms)Closed at 1731341711879 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731341702648.b7e69cf109199b3c3d754201ea9818cd. 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ba7b9cc976155fbb76bf00c2ffd4495, disabling compactions & flushes 2024-11-11T16:15:11,879 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. after waiting 0 ms 2024-11-11T16:15:11,879 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:11,881 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T16:15:11,881 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T16:15:11,882 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ee84052923dbaf7f5c9581236276c4c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,883 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ee84052923dbaf7f5c9581236276c4c: Waiting for close lock at 1731341711878Running coprocessor pre-close hooks at 1731341711878Disabling compacts and flushes for region at 1731341711878Disabling writes for close at 1731341711878Writing region close event to WAL at 1731341711878Running coprocessor post-close hooks at 1731341711883 (+5 ms)Closed at 1731341711883 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731341702648.7ee84052923dbaf7f5c9581236276c4c. 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd1d8bee83938f0837b5cdacde7e89c4, disabling compactions & flushes 2024-11-11T16:15:11,883 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. after waiting 0 ms 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7ba7b9cc976155fbb76bf00c2ffd4495/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,884 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ba7b9cc976155fbb76bf00c2ffd4495: Waiting for close lock at 1731341711879Running coprocessor pre-close hooks at 1731341711879Disabling compacts and flushes for region at 1731341711879Disabling writes for close at 1731341711879Writing region close event to WAL at 1731341711880 (+1 ms)Running coprocessor post-close hooks at 1731341711884 (+4 ms)Closed at 1731341711884 2024-11-11T16:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731341702648.7ba7b9cc976155fbb76bf00c2ffd4495. 2024-11-11T16:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 35b42287bbbc0c5919124a8c4b800ea1, disabling compactions & flushes 2024-11-11T16:15:11,885 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:11,885 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:11,885 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. after waiting 0 ms 2024-11-11T16:15:11,885 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:11,888 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/cd1d8bee83938f0837b5cdacde7e89c4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,888 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:11,888 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/35b42287bbbc0c5919124a8c4b800ea1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,888 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd1d8bee83938f0837b5cdacde7e89c4: Waiting for close lock at 1731341711883Running coprocessor pre-close hooks at 1731341711883Disabling compacts and flushes for region at 1731341711883Disabling writes for close at 1731341711883Writing region close event to WAL at 1731341711883Running coprocessor post-close hooks at 1731341711888 (+5 ms)Closed at 1731341711888 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731341702648.cd1d8bee83938f0837b5cdacde7e89c4. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e762741692724ea54cb4ce74a7e615ae, disabling compactions & flushes 2024-11-11T16:15:11,889 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. after waiting 0 ms 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:11,889 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 35b42287bbbc0c5919124a8c4b800ea1: Waiting for close lock at 1731341711884Running coprocessor pre-close hooks at 1731341711884Disabling compacts and flushes for region at 1731341711884Disabling writes for close at 1731341711885 (+1 ms)Writing region close event to WAL at 1731341711885Running coprocessor post-close hooks at 1731341711889 (+4 ms)Closed at 1731341711889 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731341702648.35b42287bbbc0c5919124a8c4b800ea1. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e3cef8c6c3398e017e8dfa5cd0c9c242, disabling compactions & flushes 2024-11-11T16:15:11,889 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:11,889 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:11,890 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. after waiting 0 ms 2024-11-11T16:15:11,890 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:11,893 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e762741692724ea54cb4ce74a7e615ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,894 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e3cef8c6c3398e017e8dfa5cd0c9c242/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e762741692724ea54cb4ce74a7e615ae: Waiting for close lock at 1731341711889Running coprocessor pre-close hooks at 1731341711889Disabling compacts and flushes for region at 1731341711889Disabling writes for close at 1731341711889Writing region close event to WAL at 1731341711889Running coprocessor post-close hooks at 1731341711894 (+5 ms)Closed at 1731341711894 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731341702648.e762741692724ea54cb4ce74a7e615ae. 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 98ffcdd491bf651a96a2bf2c2b5a9a95, disabling compactions & flushes 2024-11-11T16:15:11,894 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. after waiting 0 ms 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:11,894 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e3cef8c6c3398e017e8dfa5cd0c9c242: Waiting for close lock at 1731341711889Running coprocessor pre-close hooks at 1731341711889Disabling compacts and flushes for region at 1731341711889Disabling writes for close at 1731341711890 (+1 ms)Writing region close event to WAL at 1731341711890Running coprocessor post-close hooks at 1731341711894 (+4 ms)Closed at 1731341711894 2024-11-11T16:15:11,894 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731341702648.e3cef8c6c3398e017e8dfa5cd0c9c242. 2024-11-11T16:15:11,895 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8845a47073d69ee0a0021d18e08e51ef, disabling compactions & flushes 2024-11-11T16:15:11,895 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:11,895 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:11,895 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. after waiting 0 ms 2024-11-11T16:15:11,895 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:11,898 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/98ffcdd491bf651a96a2bf2c2b5a9a95/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,899 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 98ffcdd491bf651a96a2bf2c2b5a9a95: Waiting for close lock at 1731341711894Running coprocessor pre-close hooks at 1731341711894Disabling compacts and flushes for region at 1731341711894Disabling writes for close at 1731341711894Writing region close event to WAL at 1731341711894Running coprocessor post-close hooks at 1731341711899 (+5 ms)Closed at 1731341711899 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/8845a47073d69ee0a0021d18e08e51ef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731341702648.98ffcdd491bf651a96a2bf2c2b5a9a95. 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e62b8abd36e22d5d589c5bdd191ed105, disabling compactions & flushes 2024-11-11T16:15:11,899 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. after waiting 0 ms 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:11,899 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:11,899 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8845a47073d69ee0a0021d18e08e51ef: Waiting for close lock at 1731341711895Running coprocessor pre-close hooks at 1731341711895Disabling compacts and flushes for region at 1731341711895Disabling writes for close at 1731341711895Writing region close event to WAL at 1731341711895Running coprocessor post-close hooks at 1731341711899 (+4 ms)Closed at 1731341711899 2024-11-11T16:15:11,900 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731341702648.8845a47073d69ee0a0021d18e08e51ef. 2024-11-11T16:15:11,900 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c2941f7d018cc329ed386329a479fbf, disabling compactions & flushes 2024-11-11T16:15:11,900 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:11,900 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:11,900 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. after waiting 0 ms 2024-11-11T16:15:11,900 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:11,902 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T16:15:11,902 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T16:15:11,903 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/e62b8abd36e22d5d589c5bdd191ed105/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,904 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:11,904 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e62b8abd36e22d5d589c5bdd191ed105: Waiting for close lock at 1731341711899Running coprocessor pre-close hooks at 1731341711899Disabling compacts and flushes for region at 1731341711899Disabling writes for close at 1731341711899Writing region close event to WAL at 1731341711899Running coprocessor post-close hooks at 1731341711904 (+5 ms)Closed at 1731341711904 2024-11-11T16:15:11,904 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731341702648.e62b8abd36e22d5d589c5bdd191ed105. 2024-11-11T16:15:11,904 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/4c2941f7d018cc329ed386329a479fbf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,905 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c2941f7d018cc329ed386329a479fbf: Waiting for close lock at 1731341711900Running coprocessor pre-close hooks at 1731341711900Disabling compacts and flushes for region at 1731341711900Disabling writes for close at 1731341711900Writing region close event to WAL at 1731341711900Running coprocessor post-close hooks at 1731341711905 (+5 ms)Closed at 1731341711905 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731341702648.4c2941f7d018cc329ed386329a479fbf. 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7df81ce6daca1d892d80d840de0c0b8b, disabling compactions & flushes 2024-11-11T16:15:11,905 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. after waiting 0 ms 2024-11-11T16:15:11,905 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:11,909 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/data/default/test/7df81ce6daca1d892d80d840de0c0b8b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:11,910 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:11,910 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7df81ce6daca1d892d80d840de0c0b8b: Waiting for close lock at 1731341711905Running coprocessor pre-close hooks at 1731341711905Disabling compacts and flushes for region at 1731341711905Disabling writes for close at 1731341711905Writing region close event to WAL at 1731341711905Running coprocessor post-close hooks at 1731341711909 (+4 ms)Closed at 1731341711909 2024-11-11T16:15:11,910 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731341702648.7df81ce6daca1d892d80d840de0c0b8b. 2024-11-11T16:15:11,962 INFO [RS:0;16b413a53992:44579 {}] regionserver.HRegionServer(976): stopping server 16b413a53992,44579,1731341698497; all regions closed. 2024-11-11T16:15:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741834_1010 (size=7064) 2024-11-11T16:15:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741834_1010 (size=7064) 2024-11-11T16:15:11,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741834_1010 (size=7064) 2024-11-11T16:15:11,969 INFO [RS:1;16b413a53992:33497 {}] regionserver.HRegionServer(976): stopping server 16b413a53992,33497,1731341698716; all regions closed. 2024-11-11T16:15:11,970 DEBUG [RS:0;16b413a53992:44579 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/oldWALs 2024-11-11T16:15:11,970 INFO [RS:0;16b413a53992:44579 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b413a53992%2C44579%2C1731341698497:(num 1731341701102) 2024-11-11T16:15:11,970 DEBUG [RS:0;16b413a53992:44579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,970 INFO [RS:0;16b413a53992:44579 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] hbase.ChoreService(370): Chore service for: regionserver/16b413a53992:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T16:15:11,971 INFO [RS:0;16b413a53992:44579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T16:15:11,972 INFO [RS:0;16b413a53992:44579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44579 2024-11-11T16:15:11,972 INFO [regionserver/16b413a53992:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T16:15:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741865_1041 (size=4201) 2024-11-11T16:15:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741865_1041 (size=4201) 2024-11-11T16:15:11,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741865_1041 (size=4201) 2024-11-11T16:15:11,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44579-0x1002f9fc70b0001, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b413a53992,44579,1731341698497 2024-11-11T16:15:11,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:15:11,977 INFO [RS:0;16b413a53992:44579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T16:15:11,979 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b413a53992,44579,1731341698497] 2024-11-11T16:15:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741833_1009 (size=6536) 2024-11-11T16:15:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741833_1009 (size=6536) 2024-11-11T16:15:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741833_1009 (size=6536) 2024-11-11T16:15:11,980 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b413a53992,44579,1731341698497 already deleted, retry=false 2024-11-11T16:15:11,981 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b413a53992,44579,1731341698497 expired; onlineServers=1 2024-11-11T16:15:11,981 DEBUG [RS:1;16b413a53992:33497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:11,981 INFO [RS:1;16b413a53992:33497 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:15:11,981 INFO [RS:1;16b413a53992:33497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T16:15:11,981 INFO [RS:1;16b413a53992:33497 {}] hbase.ChoreService(370): Chore service for: regionserver/16b413a53992:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T16:15:11,981 INFO [RS:1;16b413a53992:33497 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T16:15:11,981 INFO [regionserver/16b413a53992:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T16:15:11,982 INFO [RS:1;16b413a53992:33497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33497 2024-11-11T16:15:11,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33497-0x1002f9fc70b0002, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b413a53992,33497,1731341698716 2024-11-11T16:15:11,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:15:11,984 INFO [RS:1;16b413a53992:33497 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T16:15:11,985 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b413a53992,33497,1731341698716] 2024-11-11T16:15:11,986 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b413a53992,33497,1731341698716 already deleted, retry=false 2024-11-11T16:15:11,986 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b413a53992,33497,1731341698716 expired; onlineServers=0 2024-11-11T16:15:11,987 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16b413a53992,35895,1731341697364' ***** 2024-11-11T16:15:11,987 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T16:15:11,987 INFO [M:0;16b413a53992:35895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:15:11,987 INFO [M:0;16b413a53992:35895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T16:15:11,987 DEBUG [M:0;16b413a53992:35895 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T16:15:11,987 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T16:15:11,987 DEBUG [master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341700627 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341700627,5,FailOnTimeoutGroup] 2024-11-11T16:15:11,987 DEBUG [master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341700657 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341700657,5,FailOnTimeoutGroup] 2024-11-11T16:15:11,987 DEBUG [M:0;16b413a53992:35895 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T16:15:11,987 INFO [M:0;16b413a53992:35895 {}] hbase.ChoreService(370): Chore service for: master/16b413a53992:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T16:15:11,988 INFO [M:0;16b413a53992:35895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T16:15:11,988 DEBUG [M:0;16b413a53992:35895 {}] master.HMaster(1795): Stopping service threads 2024-11-11T16:15:11,988 INFO [M:0;16b413a53992:35895 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T16:15:11,988 INFO [M:0;16b413a53992:35895 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:15:11,988 INFO [M:0;16b413a53992:35895 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:15:11,988 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T16:15:11,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T16:15:11,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:15:11,989 DEBUG [M:0;16b413a53992:35895 {}] zookeeper.ZKUtil(347): master:35895-0x1002f9fc70b0000, quorum=127.0.0.1:54959, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T16:15:11,989 WARN [M:0;16b413a53992:35895 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T16:15:11,990 INFO [M:0;16b413a53992:35895 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/.lastflushedseqids 2024-11-11T16:15:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741870_1046 (size=1) 2024-11-11T16:15:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741870_1046 (size=1) 2024-11-11T16:15:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741870_1046 (size=1) 2024-11-11T16:15:12,003 INFO [M:0;16b413a53992:35895 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:15:12,003 INFO [M:0;16b413a53992:35895 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T16:15:12,003 DEBUG [M:0;16b413a53992:35895 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T16:15:12,003 INFO [M:0;16b413a53992:35895 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:15:12,003 DEBUG [M:0;16b413a53992:35895 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:15:12,003 DEBUG [M:0;16b413a53992:35895 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T16:15:12,004 DEBUG [M:0;16b413a53992:35895 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:15:12,004 INFO [M:0;16b413a53992:35895 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.49 KB heapSize=315.40 KB 2024-11-11T16:15:12,023 DEBUG [M:0;16b413a53992:35895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8c293f265af483da044bc5c9bfcb070 is 82, key is hbase:meta,,1/info:regioninfo/1731341710033/Put/seqid=0 2024-11-11T16:15:12,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37169 is added to blk_1073741871_1047 (size=6071) 2024-11-11T16:15:12,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45315 is added to blk_1073741871_1047 (size=6071) 2024-11-11T16:15:12,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40759 is added to blk_1073741871_1047 (size=6071) 2024-11-11T16:15:12,034 INFO [M:0;16b413a53992:35895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8c293f265af483da044bc5c9bfcb070 2024-11-11T16:15:12,063 DEBUG [M:0;16b413a53992:35895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34433/user/jenkins/test-data/aac6d8e6-4407-f3e9-659e-3175c3158dba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/89c523e373ad44ada3d3afce1f3889d2 is 2987, key is \x00\x00\x00\x00\x00\x00\x00