2024-11-07 21:13:29,061 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-07 21:13:29,074 main DEBUG Took 0.011404 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 21:13:29,075 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 21:13:29,075 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 21:13:29,076 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 21:13:29,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,085 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 21:13:29,100 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,103 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,104 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,105 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,106 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,107 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,108 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,109 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,110 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,111 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,112 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 21:13:29,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,114 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 21:13:29,116 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 21:13:29,117 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 21:13:29,119 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 21:13:29,120 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 21:13:29,122 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 21:13:29,122 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 21:13:29,130 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 21:13:29,133 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 21:13:29,135 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 21:13:29,135 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 21:13:29,136 main DEBUG createAppenders(={Console}) 2024-11-07 21:13:29,136 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-07 21:13:29,137 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-07 21:13:29,137 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-07 21:13:29,138 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 21:13:29,138 main DEBUG OutputStream closed 2024-11-07 21:13:29,139 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 21:13:29,139 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 21:13:29,140 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-07 21:13:29,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 21:13:29,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 21:13:29,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 21:13:29,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 21:13:29,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 21:13:29,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 21:13:29,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 21:13:29,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 21:13:29,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 21:13:29,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 21:13:29,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 21:13:29,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 21:13:29,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 21:13:29,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 21:13:29,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 21:13:29,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 21:13:29,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 21:13:29,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 21:13:29,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 21:13:29,240 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-07 21:13:29,240 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 21:13:29,240 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-07T21:13:29,251 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-07 21:13:29,254 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 21:13:29,254 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T21:13:29,477 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc 2024-11-07T21:13:29,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T21:13:29,510 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381, deleteOnExit=true 2024-11-07T21:13:29,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T21:13:29,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/test.cache.data in system properties and HBase conf 2024-11-07T21:13:29,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T21:13:29,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.log.dir in system properties and HBase conf 2024-11-07T21:13:29,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T21:13:29,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T21:13:29,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T21:13:29,600 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T21:13:29,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T21:13:29,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T21:13:29,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T21:13:29,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T21:13:29,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T21:13:29,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T21:13:29,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T21:13:29,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T21:13:29,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T21:13:29,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T21:13:29,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/nfs.dump.dir in system properties and HBase conf 2024-11-07T21:13:29,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/java.io.tmpdir in system properties and HBase conf 2024-11-07T21:13:29,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T21:13:29,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T21:13:29,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T21:13:30,792 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T21:13:30,858 INFO [Time-limited test {}] log.Log(170): Logging initialized @2465ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T21:13:30,922 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T21:13:30,975 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T21:13:31,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T21:13:31,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T21:13:31,002 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T21:13:31,017 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T21:13:31,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.log.dir/,AVAILABLE} 2024-11-07T21:13:31,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T21:13:31,215 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/java.io.tmpdir/jetty-localhost-46237-hadoop-hdfs-3_4_1-tests_jar-_-any-7694382971665316488/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T21:13:31,226 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:46237} 2024-11-07T21:13:31,226 INFO [Time-limited test {}] server.Server(415): Started @2834ms 2024-11-07T21:13:31,804 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T21:13:31,812 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T21:13:31,813 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T21:13:31,813 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T21:13:31,813 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T21:13:31,814 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.log.dir/,AVAILABLE} 2024-11-07T21:13:31,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T21:13:31,913 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/java.io.tmpdir/jetty-localhost-38867-hadoop-hdfs-3_4_1-tests_jar-_-any-9087799731557364392/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T21:13:31,913 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:38867} 2024-11-07T21:13:31,913 INFO [Time-limited test {}] server.Server(415): Started @3521ms 2024-11-07T21:13:31,961 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T21:13:32,067 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T21:13:32,075 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T21:13:32,076 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T21:13:32,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T21:13:32,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T21:13:32,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.log.dir/,AVAILABLE} 2024-11-07T21:13:32,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T21:13:32,217 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/java.io.tmpdir/jetty-localhost-46353-hadoop-hdfs-3_4_1-tests_jar-_-any-9926916119664437710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T21:13:32,217 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:46353} 2024-11-07T21:13:32,218 INFO [Time-limited test {}] server.Server(415): Started @3826ms 2024-11-07T21:13:32,220 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T21:13:32,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T21:13:32,263 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T21:13:32,264 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T21:13:32,264 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T21:13:32,264 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T21:13:32,268 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/hadoop.log.dir/,AVAILABLE} 2024-11-07T21:13:32,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T21:13:32,366 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/java.io.tmpdir/jetty-localhost-37931-hadoop-hdfs-3_4_1-tests_jar-_-any-404084766176258804/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T21:13:32,367 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:37931} 2024-11-07T21:13:32,367 INFO [Time-limited test {}] server.Server(415): Started @3975ms 2024-11-07T21:13:32,369 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T21:13:34,402 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data1/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,402 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data2/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,402 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data4/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,402 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data3/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,441 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T21:13:34,441 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T21:13:34,484 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5aae698d61f8c07 with lease ID 0x28b98e1c0afa801d: Processing first storage report for DS-1028e527-7199-40c5-8ad9-53e5043a22d0 from datanode DatanodeRegistration(127.0.0.1:42963, datanodeUuid=74aab349-cbb0-4006-8885-d6013e94c9b2, infoPort=44915, infoSecurePort=0, ipcPort=43347, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,485 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5aae698d61f8c07 with lease ID 0x28b98e1c0afa801d: from storage DS-1028e527-7199-40c5-8ad9-53e5043a22d0 node DatanodeRegistration(127.0.0.1:42963, datanodeUuid=74aab349-cbb0-4006-8885-d6013e94c9b2, infoPort=44915, infoSecurePort=0, ipcPort=43347, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24efb6c4d24d09b7 with lease ID 0x28b98e1c0afa801c: Processing first storage report for DS-2b9b3999-346f-42a4-a048-4dc08136d4e0 from datanode DatanodeRegistration(127.0.0.1:44057, datanodeUuid=40fe4c68-890c-473d-8bca-51fdb9373393, infoPort=34923, infoSecurePort=0, ipcPort=35445, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24efb6c4d24d09b7 with lease ID 0x28b98e1c0afa801c: from storage DS-2b9b3999-346f-42a4-a048-4dc08136d4e0 node DatanodeRegistration(127.0.0.1:44057, datanodeUuid=40fe4c68-890c-473d-8bca-51fdb9373393, infoPort=34923, infoSecurePort=0, ipcPort=35445, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5aae698d61f8c07 with lease ID 0x28b98e1c0afa801d: Processing first storage report for DS-8281466d-043c-49c8-a5f7-66eb7dcca858 from datanode DatanodeRegistration(127.0.0.1:42963, datanodeUuid=74aab349-cbb0-4006-8885-d6013e94c9b2, infoPort=44915, infoSecurePort=0, ipcPort=43347, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5aae698d61f8c07 with lease ID 0x28b98e1c0afa801d: from storage DS-8281466d-043c-49c8-a5f7-66eb7dcca858 node DatanodeRegistration(127.0.0.1:42963, datanodeUuid=74aab349-cbb0-4006-8885-d6013e94c9b2, infoPort=44915, infoSecurePort=0, ipcPort=43347, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24efb6c4d24d09b7 with lease ID 0x28b98e1c0afa801c: Processing first storage report for DS-b60325f5-87ea-4d41-a9d3-3b19202ceb41 from datanode DatanodeRegistration(127.0.0.1:44057, datanodeUuid=40fe4c68-890c-473d-8bca-51fdb9373393, infoPort=34923, infoSecurePort=0, ipcPort=35445, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24efb6c4d24d09b7 with lease ID 0x28b98e1c0afa801c: from storage DS-b60325f5-87ea-4d41-a9d3-3b19202ceb41 node DatanodeRegistration(127.0.0.1:44057, datanodeUuid=40fe4c68-890c-473d-8bca-51fdb9373393, infoPort=34923, infoSecurePort=0, ipcPort=35445, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,680 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data5/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,680 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/data/data6/current/BP-551865938-172.17.0.2-1731014010267/current, will proceed with Du for space computation calculation, 2024-11-07T21:13:34,700 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T21:13:34,704 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82c7b8c8b96bebfd with lease ID 0x28b98e1c0afa801e: Processing first storage report for DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772 from datanode DatanodeRegistration(127.0.0.1:45187, datanodeUuid=e4a8b98a-3322-4415-8564-f8554b940c48, infoPort=37905, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,705 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82c7b8c8b96bebfd with lease ID 0x28b98e1c0afa801e: from storage DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772 node DatanodeRegistration(127.0.0.1:45187, datanodeUuid=e4a8b98a-3322-4415-8564-f8554b940c48, infoPort=37905, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82c7b8c8b96bebfd with lease ID 0x28b98e1c0afa801e: Processing first storage report for DS-f64a3334-33c9-49b0-9539-123f729cf101 from datanode DatanodeRegistration(127.0.0.1:45187, datanodeUuid=e4a8b98a-3322-4415-8564-f8554b940c48, infoPort=37905, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267) 2024-11-07T21:13:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82c7b8c8b96bebfd with lease ID 0x28b98e1c0afa801e: from storage DS-f64a3334-33c9-49b0-9539-123f729cf101 node DatanodeRegistration(127.0.0.1:45187, datanodeUuid=e4a8b98a-3322-4415-8564-f8554b940c48, infoPort=37905, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=806379923;c=1731014010267), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T21:13:34,789 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc 2024-11-07T21:13:34,852 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/zookeeper_0, clientPort=62874, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e356a4a4-735d-2b7a-32bd-641e7344b4cc/cluster_2f0878b6-a48d-dd88-92b5-63229fde5381/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T21:13:34,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62874 2024-11-07T21:13:34,870 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:34,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:35,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741825_1001 (size=7) 2024-11-07T21:13:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741825_1001 (size=7) 2024-11-07T21:13:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741825_1001 (size=7) 2024-11-07T21:13:35,488 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af with version=8 2024-11-07T21:13:35,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/hbase-staging 2024-11-07T21:13:35,568 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T21:13:35,802 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4fc742968c91:0 server-side Connection retries=45 2024-11-07T21:13:35,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:35,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:35,815 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T21:13:35,816 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:35,816 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T21:13:35,937 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T21:13:35,986 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T21:13:35,994 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T21:13:35,997 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T21:13:36,020 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3727 (auto-detected) 2024-11-07T21:13:36,020 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T21:13:36,037 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37625 2024-11-07T21:13:36,056 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37625 connecting to ZooKeeper ensemble=127.0.0.1:62874 2024-11-07T21:13:36,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376250x0, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T21:13:36,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37625-0x1011725b3200000 connected 2024-11-07T21:13:36,255 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,261 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,292 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:36,300 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af, hbase.cluster.distributed=false 2024-11-07T21:13:36,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T21:13:36,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37625 2024-11-07T21:13:36,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37625 2024-11-07T21:13:36,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37625 2024-11-07T21:13:36,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37625 2024-11-07T21:13:36,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37625 2024-11-07T21:13:36,444 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4fc742968c91:0 server-side Connection retries=45 2024-11-07T21:13:36,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,446 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T21:13:36,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,446 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T21:13:36,449 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T21:13:36,452 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T21:13:36,453 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35389 2024-11-07T21:13:36,454 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35389 connecting to ZooKeeper ensemble=127.0.0.1:62874 2024-11-07T21:13:36,455 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353890x0, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T21:13:36,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353890x0, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:36,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35389-0x1011725b3200001 connected 2024-11-07T21:13:36,475 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T21:13:36,483 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T21:13:36,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T21:13:36,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T21:13:36,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35389 2024-11-07T21:13:36,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35389 2024-11-07T21:13:36,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35389 2024-11-07T21:13:36,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35389 2024-11-07T21:13:36,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35389 2024-11-07T21:13:36,510 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4fc742968c91:0 server-side Connection retries=45 2024-11-07T21:13:36,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,511 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T21:13:36,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T21:13:36,512 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T21:13:36,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T21:13:36,513 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38729 2024-11-07T21:13:36,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38729 connecting to ZooKeeper ensemble=127.0.0.1:62874 2024-11-07T21:13:36,516 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,518 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387290x0, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T21:13:36,534 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387290x0, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:36,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38729-0x1011725b3200002 connected 2024-11-07T21:13:36,535 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T21:13:36,536 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T21:13:36,537 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T21:13:36,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T21:13:36,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-07T21:13:36,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38729 2024-11-07T21:13:36,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38729 2024-11-07T21:13:36,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-07T21:13:36,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-07T21:13:36,569 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4fc742968c91:0 server-side Connection retries=45 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T21:13:36,570 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T21:13:36,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T21:13:36,571 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38103 2024-11-07T21:13:36,573 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38103 connecting to ZooKeeper ensemble=127.0.0.1:62874 2024-11-07T21:13:36,574 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,577 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381030x0, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T21:13:36,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381030x0, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:36,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38103-0x1011725b3200003 connected 2024-11-07T21:13:36,588 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T21:13:36,589 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T21:13:36,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T21:13:36,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T21:13:36,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38103 2024-11-07T21:13:36,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38103 2024-11-07T21:13:36,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38103 2024-11-07T21:13:36,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38103 2024-11-07T21:13:36,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38103 2024-11-07T21:13:36,612 DEBUG [M:0;4fc742968c91:37625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4fc742968c91:37625 2024-11-07T21:13:36,613 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4fc742968c91,37625,1731014015654 2024-11-07T21:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,633 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4fc742968c91,37625,1731014015654 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,661 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T21:13:36,662 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4fc742968c91,37625,1731014015654 from backup master directory 2024-11-07T21:13:36,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4fc742968c91,37625,1731014015654 2024-11-07T21:13:36,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T21:13:36,671 WARN [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T21:13:36,671 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4fc742968c91,37625,1731014015654 2024-11-07T21:13:36,673 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T21:13:36,674 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T21:13:36,725 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/hbase.id] with ID: bf617550-7f62-4778-941c-0e640e550d51 2024-11-07T21:13:36,726 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/.tmp/hbase.id 2024-11-07T21:13:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741826_1002 (size=42) 2024-11-07T21:13:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741826_1002 (size=42) 2024-11-07T21:13:36,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741826_1002 (size=42) 2024-11-07T21:13:36,741 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/.tmp/hbase.id]:[hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/hbase.id] 2024-11-07T21:13:36,782 INFO [master/4fc742968c91:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T21:13:36,788 INFO [master/4fc742968c91:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T21:13:36,805 INFO [master/4fc742968c91:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-07T21:13:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:36,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741827_1003 (size=196) 2024-11-07T21:13:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741827_1003 (size=196) 2024-11-07T21:13:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741827_1003 (size=196) 2024-11-07T21:13:36,854 INFO [master/4fc742968c91:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T21:13:36,855 INFO [master/4fc742968c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T21:13:36,869 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:36,872 INFO [master/4fc742968c91:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:36,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741828_1004 (size=1189) 2024-11-07T21:13:36,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741828_1004 (size=1189) 2024-11-07T21:13:36,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741828_1004 (size=1189) 2024-11-07T21:13:36,917 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store 2024-11-07T21:13:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741829_1005 (size=34) 2024-11-07T21:13:36,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741829_1005 (size=34) 2024-11-07T21:13:36,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741829_1005 (size=34) 2024-11-07T21:13:36,943 INFO [master/4fc742968c91:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T21:13:36,947 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:36,948 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T21:13:36,948 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:36,949 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:36,950 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T21:13:36,951 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:36,951 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:36,952 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731014016948Disabling compacts and flushes for region at 1731014016948Disabling writes for close at 1731014016950 (+2 ms)Writing region close event to WAL at 1731014016951 (+1 ms)Closed at 1731014016951 2024-11-07T21:13:36,954 WARN [master/4fc742968c91:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/.initializing 2024-11-07T21:13:36,954 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/WALs/4fc742968c91,37625,1731014015654 2024-11-07T21:13:36,963 INFO [master/4fc742968c91:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T21:13:36,978 INFO [master/4fc742968c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C37625%2C1731014015654, suffix=, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/WALs/4fc742968c91,37625,1731014015654, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/oldWALs, maxLogs=10 2024-11-07T21:13:36,998 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/WALs/4fc742968c91,37625,1731014015654/4fc742968c91%2C37625%2C1731014015654.1731014016982, exclude list is [], retry=0 2024-11-07T21:13:37,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:37,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:37,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T21:13:37,051 INFO [master/4fc742968c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/WALs/4fc742968c91,37625,1731014015654/4fc742968c91%2C37625%2C1731014015654.1731014016982 2024-11-07T21:13:37,052 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44915:44915),(127.0.0.1/127.0.0.1:34923:34923),(127.0.0.1/127.0.0.1:37905:37905)] 2024-11-07T21:13:37,053 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T21:13:37,053 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:37,056 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,057 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T21:13:37,116 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:37,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T21:13:37,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:37,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T21:13:37,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:37,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T21:13:37,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:37,131 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,135 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,136 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,141 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,141 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,145 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T21:13:37,150 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T21:13:37,155 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:37,156 INFO [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69050997, jitterRate=0.02894003689289093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T21:13:37,163 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731014017068Initializing all the Stores at 1731014017070 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014017070Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014017071 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014017071Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014017071Cleaning up temporary data from old regions at 1731014017141 (+70 ms)Region opened successfully at 1731014017163 (+22 ms) 2024-11-07T21:13:37,164 INFO [master/4fc742968c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T21:13:37,196 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638902e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fc742968c91/172.17.0.2:0 2024-11-07T21:13:37,228 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T21:13:37,239 INFO [master/4fc742968c91:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:37,239 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:37,242 INFO [master/4fc742968c91:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:37,243 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T21:13:37,247 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T21:13:37,248 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T21:13:37,271 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T21:13:37,279 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T21:13:37,323 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T21:13:37,326 INFO [master/4fc742968c91:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T21:13:37,329 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T21:13:37,333 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T21:13:37,337 INFO [master/4fc742968c91:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T21:13:37,341 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T21:13:37,354 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T21:13:37,356 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T21:13:37,365 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T21:13:37,386 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T21:13:37,396 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T21:13:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,413 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4fc742968c91,37625,1731014015654, sessionid=0x1011725b3200000, setting cluster-up flag (Was=false) 2024-11-07T21:13:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,471 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T21:13:37,473 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4fc742968c91,37625,1731014015654 2024-11-07T21:13:37,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:37,523 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T21:13:37,526 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4fc742968c91,37625,1731014015654 2024-11-07T21:13:37,535 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T21:13:37,598 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(746): ClusterId : bf617550-7f62-4778-941c-0e640e550d51 2024-11-07T21:13:37,598 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(746): ClusterId : bf617550-7f62-4778-941c-0e640e550d51 2024-11-07T21:13:37,598 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(746): ClusterId : bf617550-7f62-4778-941c-0e640e550d51 2024-11-07T21:13:37,600 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T21:13:37,600 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T21:13:37,600 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T21:13:37,600 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:37,609 INFO [master/4fc742968c91:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T21:13:37,615 INFO [master/4fc742968c91:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T21:13:37,619 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T21:13:37,619 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T21:13:37,619 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T21:13:37,619 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T21:13:37,619 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T21:13:37,619 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T21:13:37,622 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4fc742968c91,37625,1731014015654 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T21:13:37,629 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T21:13:37,629 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T21:13:37,629 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T21:13:37,630 DEBUG [RS:1;4fc742968c91:38729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f8a636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fc742968c91/172.17.0.2:0 2024-11-07T21:13:37,630 DEBUG [RS:2;4fc742968c91:38103 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e90c6d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fc742968c91/172.17.0.2:0 2024-11-07T21:13:37,630 DEBUG [RS:0;4fc742968c91:35389 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d261d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fc742968c91/172.17.0.2:0 2024-11-07T21:13:37,631 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4fc742968c91:0, corePoolSize=5, maxPoolSize=5 2024-11-07T21:13:37,631 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4fc742968c91:0, corePoolSize=5, maxPoolSize=5 2024-11-07T21:13:37,631 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4fc742968c91:0, corePoolSize=5, maxPoolSize=5 2024-11-07T21:13:37,631 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4fc742968c91:0, corePoolSize=5, maxPoolSize=5 2024-11-07T21:13:37,632 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4fc742968c91:0, corePoolSize=10, maxPoolSize=10 2024-11-07T21:13:37,632 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,632 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4fc742968c91:0, corePoolSize=2, maxPoolSize=2 2024-11-07T21:13:37,632 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,636 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014047636 2024-11-07T21:13:37,638 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T21:13:37,639 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T21:13:37,642 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:37,642 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T21:13:37,643 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T21:13:37,644 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T21:13:37,644 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T21:13:37,644 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T21:13:37,649 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4fc742968c91:35389 2024-11-07T21:13:37,649 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,649 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4fc742968c91:38103 2024-11-07T21:13:37,649 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4fc742968c91:38729 2024-11-07T21:13:37,649 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T21:13:37,648 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,651 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T21:13:37,652 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T21:13:37,653 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T21:13:37,653 INFO [RS:1;4fc742968c91:38729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T21:13:37,653 INFO [RS:0;4fc742968c91:35389 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T21:13:37,653 INFO [RS:2;4fc742968c91:38103 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T21:13:37,653 INFO [RS:2;4fc742968c91:38103 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T21:13:37,653 INFO [RS:1;4fc742968c91:38729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T21:13:37,653 INFO [RS:0;4fc742968c91:35389 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T21:13:37,653 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T21:13:37,653 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T21:13:37,653 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T21:13:37,655 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T21:13:37,656 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(2659): reportForDuty to master=4fc742968c91,37625,1731014015654 with port=38103, startcode=1731014016569 2024-11-07T21:13:37,656 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(2659): reportForDuty to master=4fc742968c91,37625,1731014015654 with port=38729, startcode=1731014016510 2024-11-07T21:13:37,656 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(2659): reportForDuty to master=4fc742968c91,37625,1731014015654 with port=35389, startcode=1731014016411 2024-11-07T21:13:37,656 INFO [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T21:13:37,664 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.large.0-1731014017657,5,FailOnTimeoutGroup] 2024-11-07T21:13:37,668 DEBUG [RS:2;4fc742968c91:38103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T21:13:37,668 DEBUG [RS:0;4fc742968c91:35389 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T21:13:37,668 DEBUG [RS:1;4fc742968c91:38729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T21:13:37,669 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.small.0-1731014017664,5,FailOnTimeoutGroup] 2024-11-07T21:13:37,669 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,669 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T21:13:37,671 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,671 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741831_1007 (size=1321) 2024-11-07T21:13:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741831_1007 (size=1321) 2024-11-07T21:13:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741831_1007 (size=1321) 2024-11-07T21:13:37,681 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T21:13:37,682 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:37,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741832_1008 (size=32) 2024-11-07T21:13:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741832_1008 (size=32) 2024-11-07T21:13:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741832_1008 (size=32) 2024-11-07T21:13:37,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:37,732 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T21:13:37,732 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32893, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T21:13:37,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T21:13:37,733 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40565, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T21:13:37,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T21:13:37,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:37,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T21:13:37,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T21:13:37,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,741 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:37,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T21:13:37,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(517): Registering regionserver=4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T21:13:37,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:37,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T21:13:37,751 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T21:13:37,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:37,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:37,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T21:13:37,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:37,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:37,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T21:13:37,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T21:13:37,758 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T21:13:37,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(517): Registering regionserver=4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T21:13:37,764 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:37,764 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42697 2024-11-07T21:13:37,764 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T21:13:37,767 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:37,767 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,768 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] master.ServerManager(517): Registering regionserver=4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,768 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:37,768 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42697 2024-11-07T21:13:37,768 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T21:13:37,768 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67170912, jitterRate=9.245872497558594E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T21:13:37,771 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:37,772 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42697 2024-11-07T21:13:37,772 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T21:13:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731014017730Initializing all the Stores at 1731014017732 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014017732Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014017732Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014017732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014017732Cleaning up temporary data from old regions at 1731014017757 (+25 ms)Region opened successfully at 1731014017773 (+16 ms) 2024-11-07T21:13:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T21:13:37,773 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T21:13:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T21:13:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T21:13:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T21:13:37,775 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T21:13:37,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731014017773Disabling compacts and flushes for region at 1731014017773Disabling writes for close at 1731014017773Writing region close event to WAL at 1731014017774 (+1 ms)Closed at 1731014017775 (+1 ms) 2024-11-07T21:13:37,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T21:13:37,778 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:37,778 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T21:13:37,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T21:13:37,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:37,798 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T21:13:37,852 DEBUG [RS:2;4fc742968c91:38103 {}] zookeeper.ZKUtil(111): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,852 WARN [RS:2;4fc742968c91:38103 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T21:13:37,852 INFO [RS:2;4fc742968c91:38103 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:37,853 DEBUG [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,854 DEBUG [RS:0;4fc742968c91:35389 {}] zookeeper.ZKUtil(111): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,854 DEBUG [RS:1;4fc742968c91:38729 {}] zookeeper.ZKUtil(111): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,854 WARN [RS:0;4fc742968c91:35389 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T21:13:37,854 WARN [RS:1;4fc742968c91:38729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T21:13:37,855 INFO [RS:0;4fc742968c91:35389 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:37,855 INFO [RS:1;4fc742968c91:38729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:37,855 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4fc742968c91,38729,1731014016510] 2024-11-07T21:13:37,855 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4fc742968c91,38103,1731014016569] 2024-11-07T21:13:37,855 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,855 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,855 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4fc742968c91,35389,1731014016411] 2024-11-07T21:13:37,882 INFO [RS:0;4fc742968c91:35389 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T21:13:37,882 INFO [RS:1;4fc742968c91:38729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T21:13:37,882 INFO [RS:2;4fc742968c91:38103 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T21:13:37,900 INFO [RS:1;4fc742968c91:38729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T21:13:37,900 INFO [RS:0;4fc742968c91:35389 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T21:13:37,900 INFO [RS:2;4fc742968c91:38103 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T21:13:37,904 INFO [RS:0;4fc742968c91:35389 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T21:13:37,904 INFO [RS:2;4fc742968c91:38103 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T21:13:37,904 INFO [RS:1;4fc742968c91:38729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T21:13:37,905 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,905 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,905 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,906 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T21:13:37,906 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T21:13:37,906 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T21:13:37,911 INFO [RS:1;4fc742968c91:38729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T21:13:37,911 INFO [RS:2;4fc742968c91:38103 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T21:13:37,911 INFO [RS:0;4fc742968c91:35389 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T21:13:37,913 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,913 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,913 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,913 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,913 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,913 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,914 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0, corePoolSize=2, maxPoolSize=2 2024-11-07T21:13:37,914 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0, corePoolSize=2, maxPoolSize=2 2024-11-07T21:13:37,914 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0, corePoolSize=2, maxPoolSize=2 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,915 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,915 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,916 DEBUG [RS:2;4fc742968c91:38103 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,916 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4fc742968c91:0, corePoolSize=1, maxPoolSize=1 2024-11-07T21:13:37,916 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,916 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,916 DEBUG [RS:1;4fc742968c91:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,916 DEBUG [RS:0;4fc742968c91:35389 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4fc742968c91:0, corePoolSize=3, maxPoolSize=3 2024-11-07T21:13:37,918 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,38729,1731014016510-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,38103,1731014016569-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T21:13:37,919 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,35389,1731014016411-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T21:13:37,938 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T21:13:37,940 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T21:13:37,941 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,38729,1731014016510-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,941 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,35389,1731014016411-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,941 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,941 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,941 INFO [RS:0;4fc742968c91:35389 {}] regionserver.Replication(171): 4fc742968c91,35389,1731014016411 started 2024-11-07T21:13:37,941 INFO [RS:1;4fc742968c91:38729 {}] regionserver.Replication(171): 4fc742968c91,38729,1731014016510 started 2024-11-07T21:13:37,946 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T21:13:37,946 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,38103,1731014016569-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,946 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,947 INFO [RS:2;4fc742968c91:38103 {}] regionserver.Replication(171): 4fc742968c91,38103,1731014016569 started 2024-11-07T21:13:37,949 WARN [4fc742968c91:37625 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-07T21:13:37,967 INFO [RS:2;4fc742968c91:38103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,967 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,967 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:37,967 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1482): Serving as 4fc742968c91,38103,1731014016569, RpcServer on 4fc742968c91/172.17.0.2:38103, sessionid=0x1011725b3200003 2024-11-07T21:13:37,967 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1482): Serving as 4fc742968c91,38729,1731014016510, RpcServer on 4fc742968c91/172.17.0.2:38729, sessionid=0x1011725b3200002 2024-11-07T21:13:37,967 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1482): Serving as 4fc742968c91,35389,1731014016411, RpcServer on 4fc742968c91/172.17.0.2:35389, sessionid=0x1011725b3200001 2024-11-07T21:13:37,968 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T21:13:37,968 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T21:13:37,968 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T21:13:37,968 DEBUG [RS:2;4fc742968c91:38103 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,968 DEBUG [RS:1;4fc742968c91:38729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,968 DEBUG [RS:0;4fc742968c91:35389 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,968 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,38103,1731014016569' 2024-11-07T21:13:37,968 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,38729,1731014016510' 2024-11-07T21:13:37,968 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,35389,1731014016411' 2024-11-07T21:13:37,969 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T21:13:37,969 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T21:13:37,969 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T21:13:37,970 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T21:13:37,970 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T21:13:37,970 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T21:13:37,970 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T21:13:37,970 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T21:13:37,970 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T21:13:37,970 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T21:13:37,970 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T21:13:37,970 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T21:13:37,971 DEBUG [RS:0;4fc742968c91:35389 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4fc742968c91,35389,1731014016411 2024-11-07T21:13:37,971 DEBUG [RS:2;4fc742968c91:38103 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4fc742968c91,38103,1731014016569 2024-11-07T21:13:37,971 DEBUG [RS:1;4fc742968c91:38729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4fc742968c91,38729,1731014016510 2024-11-07T21:13:37,971 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,38103,1731014016569' 2024-11-07T21:13:37,971 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,35389,1731014016411' 2024-11-07T21:13:37,971 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fc742968c91,38729,1731014016510' 2024-11-07T21:13:37,971 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T21:13:37,971 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T21:13:37,971 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T21:13:37,971 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T21:13:37,971 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T21:13:37,971 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T21:13:37,972 DEBUG [RS:2;4fc742968c91:38103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T21:13:37,972 DEBUG [RS:0;4fc742968c91:35389 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T21:13:37,972 INFO [RS:0;4fc742968c91:35389 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T21:13:37,972 INFO [RS:2;4fc742968c91:38103 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T21:13:37,972 INFO [RS:2;4fc742968c91:38103 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T21:13:37,972 INFO [RS:0;4fc742968c91:35389 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T21:13:37,976 DEBUG [RS:1;4fc742968c91:38729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T21:13:37,976 INFO [RS:1;4fc742968c91:38729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T21:13:37,976 INFO [RS:1;4fc742968c91:38729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T21:13:38,079 INFO [RS:2;4fc742968c91:38103 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T21:13:38,079 INFO [RS:0;4fc742968c91:35389 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T21:13:38,079 INFO [RS:1;4fc742968c91:38729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T21:13:38,083 INFO [RS:1;4fc742968c91:38729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C38729%2C1731014016510, suffix=, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs, maxLogs=32 2024-11-07T21:13:38,083 INFO [RS:0;4fc742968c91:35389 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C35389%2C1731014016411, suffix=, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,35389,1731014016411, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs, maxLogs=32 2024-11-07T21:13:38,083 INFO [RS:2;4fc742968c91:38103 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C38103%2C1731014016569, suffix=, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs, maxLogs=32 2024-11-07T21:13:38,100 DEBUG [RS:0;4fc742968c91:35389 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,35389,1731014016411/4fc742968c91%2C35389%2C1731014016411.1731014018086, exclude list is [], retry=0 2024-11-07T21:13:38,100 DEBUG [RS:1;4fc742968c91:38729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510/4fc742968c91%2C38729%2C1731014016510.1731014018086, exclude list is [], retry=0 2024-11-07T21:13:38,104 DEBUG [RS:2;4fc742968c91:38103 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569/4fc742968c91%2C38103%2C1731014016569.1731014018086, exclude list is [], retry=0 2024-11-07T21:13:38,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:38,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:38,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:38,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:38,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:38,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:38,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:38,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:38,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:38,117 INFO [RS:0;4fc742968c91:35389 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,35389,1731014016411/4fc742968c91%2C35389%2C1731014016411.1731014018086 2024-11-07T21:13:38,118 INFO [RS:1;4fc742968c91:38729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510/4fc742968c91%2C38729%2C1731014016510.1731014018086 2024-11-07T21:13:38,118 DEBUG [RS:0;4fc742968c91:35389 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44915:44915),(127.0.0.1/127.0.0.1:37905:37905),(127.0.0.1/127.0.0.1:34923:34923)] 2024-11-07T21:13:38,120 DEBUG [RS:1;4fc742968c91:38729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44915:44915),(127.0.0.1/127.0.0.1:34923:34923),(127.0.0.1/127.0.0.1:37905:37905)] 2024-11-07T21:13:38,123 INFO [RS:2;4fc742968c91:38103 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569/4fc742968c91%2C38103%2C1731014016569.1731014018086 2024-11-07T21:13:38,124 DEBUG [RS:2;4fc742968c91:38103 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44915:44915),(127.0.0.1/127.0.0.1:34923:34923),(127.0.0.1/127.0.0.1:37905:37905)] 2024-11-07T21:13:38,204 DEBUG [4fc742968c91:37625 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-07T21:13:38,215 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(204): Hosts are {4fc742968c91=0} racks are {/default-rack=0} 2024-11-07T21:13:38,221 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T21:13:38,221 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T21:13:38,222 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-07T21:13:38,222 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T21:13:38,222 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T21:13:38,222 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T21:13:38,222 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T21:13:38,222 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T21:13:38,222 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T21:13:38,222 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T21:13:38,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:38,235 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38103,1731014016569, state=OPENING 2024-11-07T21:13:38,281 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T21:13:38,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:38,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:38,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:38,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:38,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,297 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:38,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:38,477 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T21:13:38,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51881, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T21:13:38,491 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T21:13:38,491 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:38,492 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T21:13:38,495 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C38103%2C1731014016569.meta, suffix=.meta, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs, maxLogs=32 2024-11-07T21:13:38,510 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, exclude list is [], retry=0 2024-11-07T21:13:38,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:38,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:38,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:38,517 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta 2024-11-07T21:13:38,518 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34923:34923),(127.0.0.1/127.0.0.1:37905:37905),(127.0.0.1/127.0.0.1:44915:44915)] 2024-11-07T21:13:38,518 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T21:13:38,520 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T21:13:38,522 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T21:13:38,526 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T21:13:38,530 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T21:13:38,531 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:38,531 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T21:13:38,531 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T21:13:38,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T21:13:38,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T21:13:38,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:38,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:38,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T21:13:38,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T21:13:38,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:38,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:38,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T21:13:38,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T21:13:38,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:38,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:38,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T21:13:38,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T21:13:38,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:38,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:38,543 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T21:13:38,545 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:38,547 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:38,550 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T21:13:38,550 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T21:13:38,551 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T21:13:38,554 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T21:13:38,556 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68834569, jitterRate=0.02571500837802887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T21:13:38,556 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T21:13:38,558 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731014018531Writing region info on filesystem at 1731014018531Initializing all the Stores at 1731014018533 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014018533Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014018534 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014018534Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014018534Cleaning up temporary data from old regions at 1731014018550 (+16 ms)Running coprocessor post-open hooks at 1731014018556 (+6 ms)Region opened successfully at 1731014018558 (+2 ms) 2024-11-07T21:13:38,564 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731014018468 2024-11-07T21:13:38,573 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T21:13:38,574 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T21:13:38,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:38,577 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38103,1731014016569, state=OPEN 2024-11-07T21:13:38,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:38,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:38,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:38,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:38,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:38,651 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4fc742968c91,38103,1731014016569 2024-11-07T21:13:38,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T21:13:38,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38103,1731014016569 in 351 msec 2024-11-07T21:13:38,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T21:13:38,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 879 msec 2024-11-07T21:13:38,672 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:38,672 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T21:13:38,688 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T21:13:38,690 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1] 2024-11-07T21:13:38,707 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T21:13:38,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53013, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T21:13:38,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1630 sec 2024-11-07T21:13:38,729 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731014018729, completionTime=-1 2024-11-07T21:13:38,733 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-07T21:13:38,733 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:38,765 INFO [master/4fc742968c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-07T21:13:38,765 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014078765 2024-11-07T21:13:38,765 INFO [master/4fc742968c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014138765 2024-11-07T21:13:38,765 INFO [master/4fc742968c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-07T21:13:38,769 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:38,776 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,776 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,776 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,778 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4fc742968c91:37625, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,778 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,778 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,786 DEBUG [master/4fc742968c91:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T21:13:38,807 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.135sec 2024-11-07T21:13:38,809 INFO [master/4fc742968c91:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T21:13:38,810 INFO [master/4fc742968c91:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T21:13:38,811 INFO [master/4fc742968c91:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T21:13:38,811 INFO [master/4fc742968c91:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T21:13:38,811 INFO [master/4fc742968c91:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T21:13:38,812 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T21:13:38,812 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T21:13:38,817 DEBUG [master/4fc742968c91:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T21:13:38,818 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T21:13:38,819 INFO [master/4fc742968c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fc742968c91,37625,1731014015654-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T21:13:38,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15320c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T21:13:38,915 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T21:13:38,915 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T21:13:38,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fc742968c91,37625,-1 for getting cluster id 2024-11-07T21:13:38,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T21:13:38,929 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bf617550-7f62-4778-941c-0e640e550d51' 2024-11-07T21:13:38,931 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T21:13:38,931 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bf617550-7f62-4778-941c-0e640e550d51" 2024-11-07T21:13:38,932 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c3ffe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T21:13:38,932 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fc742968c91,37625,-1] 2024-11-07T21:13:38,934 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T21:13:38,936 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:38,937 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T21:13:38,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a37782d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T21:13:38,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T21:13:38,946 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1] 2024-11-07T21:13:38,946 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T21:13:38,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T21:13:38,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4fc742968c91,37625,1731014015654 2024-11-07T21:13:38,994 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T21:13:39,000 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fc742968c91,37625,1731014015654 2024-11-07T21:13:39,002 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@dda9621 2024-11-07T21:13:39,003 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T21:13:39,005 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T21:13:39,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T21:13:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:39,026 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T21:13:39,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-07T21:13:39,029 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:39,032 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T21:13:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741837_1013 (size=358) 2024-11-07T21:13:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741837_1013 (size=358) 2024-11-07T21:13:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741837_1013 (size=358) 2024-11-07T21:13:39,048 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ebed7942a177c6981a4264c1bfd63ce, NAME => 'test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,049 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 488c6da65f4aeee647de08a2f68cfcc9, NAME => 'test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,049 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3136c80b02fddd6691fe6a2792b436c5, NAME => 'test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741839_1015 (size=42) 2024-11-07T21:13:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741839_1015 (size=42) 2024-11-07T21:13:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741839_1015 (size=42) 2024-11-07T21:13:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741838_1014 (size=45) 2024-11-07T21:13:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741838_1014 (size=45) 2024-11-07T21:13:39,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741838_1014 (size=45) 2024-11-07T21:13:39,073 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,073 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,073 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8ebed7942a177c6981a4264c1bfd63ce, disabling compactions & flushes 2024-11-07T21:13:39,073 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:39,073 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3136c80b02fddd6691fe6a2792b436c5, disabling compactions & flushes 2024-11-07T21:13:39,073 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:39,074 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. after waiting 0 ms 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. after waiting 0 ms 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:39,074 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:39,074 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3136c80b02fddd6691fe6a2792b436c5: Waiting for close lock at 1731014019073Disabling compacts and flushes for region at 1731014019073Disabling writes for close at 1731014019074 (+1 ms)Writing region close event to WAL at 1731014019074Closed at 1731014019074 2024-11-07T21:13:39,074 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ebed7942a177c6981a4264c1bfd63ce: Waiting for close lock at 1731014019073Disabling compacts and flushes for region at 1731014019073Disabling writes for close at 1731014019074 (+1 ms)Writing region close event to WAL at 1731014019074Closed at 1731014019074 2024-11-07T21:13:39,074 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bdc220bbb3923bf71a5ec3496dfdf44b, NAME => 'test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,075 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1e6e832a4cf4736e7c0513d0ad428d1, NAME => 'test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741840_1016 (size=45) 2024-11-07T21:13:39,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741840_1016 (size=45) 2024-11-07T21:13:39,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741840_1016 (size=45) 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 488c6da65f4aeee647de08a2f68cfcc9, disabling compactions & flushes 2024-11-07T21:13:39,088 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. after waiting 0 ms 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:39,088 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:39,088 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 488c6da65f4aeee647de08a2f68cfcc9: Waiting for close lock at 1731014019088Disabling compacts and flushes for region at 1731014019088Disabling writes for close at 1731014019088Writing region close event to WAL at 1731014019088Closed at 1731014019088 2024-11-07T21:13:39,089 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 028acc93496b53fbb3cfebe0328fb380, NAME => 'test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741841_1017 (size=45) 2024-11-07T21:13:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741841_1017 (size=45) 2024-11-07T21:13:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741841_1017 (size=45) 2024-11-07T21:13:39,104 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,104 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bdc220bbb3923bf71a5ec3496dfdf44b, disabling compactions & flushes 2024-11-07T21:13:39,104 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:39,105 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:39,105 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. after waiting 0 ms 2024-11-07T21:13:39,105 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:39,105 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:39,105 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bdc220bbb3923bf71a5ec3496dfdf44b: Waiting for close lock at 1731014019104Disabling compacts and flushes for region at 1731014019104Disabling writes for close at 1731014019105 (+1 ms)Writing region close event to WAL at 1731014019105Closed at 1731014019105 2024-11-07T21:13:39,106 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 17100f72b0290f4e0327d690648433ab, NAME => 'test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741842_1018 (size=45) 2024-11-07T21:13:39,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741842_1018 (size=45) 2024-11-07T21:13:39,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741842_1018 (size=45) 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c1e6e832a4cf4736e7c0513d0ad428d1, disabling compactions & flushes 2024-11-07T21:13:39,112 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. after waiting 0 ms 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:39,112 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:39,112 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1e6e832a4cf4736e7c0513d0ad428d1: Waiting for close lock at 1731014019112Disabling compacts and flushes for region at 1731014019112Disabling writes for close at 1731014019112Writing region close event to WAL at 1731014019112Closed at 1731014019112 2024-11-07T21:13:39,113 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 364a8945e5957c7750a8e0c67047b7a3, NAME => 'test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741843_1019 (size=45) 2024-11-07T21:13:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741843_1019 (size=45) 2024-11-07T21:13:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741843_1019 (size=45) 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 028acc93496b53fbb3cfebe0328fb380, disabling compactions & flushes 2024-11-07T21:13:39,124 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. after waiting 0 ms 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:39,124 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:39,124 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 028acc93496b53fbb3cfebe0328fb380: Waiting for close lock at 1731014019124Disabling compacts and flushes for region at 1731014019124Disabling writes for close at 1731014019124Writing region close event to WAL at 1731014019124Closed at 1731014019124 2024-11-07T21:13:39,125 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ca4a1386e839589539d875923c362933, NAME => 'test,iii,1731014019006.ca4a1386e839589539d875923c362933.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741844_1020 (size=45) 2024-11-07T21:13:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741844_1020 (size=45) 2024-11-07T21:13:39,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741844_1020 (size=45) 2024-11-07T21:13:39,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 17100f72b0290f4e0327d690648433ab, disabling compactions & flushes 2024-11-07T21:13:39,133 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:39,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:39,133 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. after waiting 0 ms 2024-11-07T21:13:39,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:39,134 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:39,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 17100f72b0290f4e0327d690648433ab: Waiting for close lock at 1731014019133Disabling compacts and flushes for region at 1731014019133Disabling writes for close at 1731014019133Writing region close event to WAL at 1731014019134 (+1 ms)Closed at 1731014019134 2024-11-07T21:13:39,134 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c9fb04694e7a790ffddb7dcec6e3022d, NAME => 'test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741845_1021 (size=45) 2024-11-07T21:13:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741845_1021 (size=45) 2024-11-07T21:13:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741845_1021 (size=45) 2024-11-07T21:13:39,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 364a8945e5957c7750a8e0c67047b7a3, disabling compactions & flushes 2024-11-07T21:13:39,149 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:39,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:39,150 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. after waiting 0 ms 2024-11-07T21:13:39,150 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:39,150 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:39,150 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 364a8945e5957c7750a8e0c67047b7a3: Waiting for close lock at 1731014019149Disabling compacts and flushes for region at 1731014019149Disabling writes for close at 1731014019150 (+1 ms)Writing region close event to WAL at 1731014019150Closed at 1731014019150 2024-11-07T21:13:39,150 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5dbe14ba56226cb0a73d7116364c76a7, NAME => 'test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741846_1022 (size=45) 2024-11-07T21:13:39,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741846_1022 (size=45) 2024-11-07T21:13:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741847_1023 (size=45) 2024-11-07T21:13:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741846_1022 (size=45) 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731014019006.ca4a1386e839589539d875923c362933.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ca4a1386e839589539d875923c362933, disabling compactions & flushes 2024-11-07T21:13:39,161 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731014019006.ca4a1386e839589539d875923c362933. after waiting 0 ms 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:39,161 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:39,161 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ca4a1386e839589539d875923c362933: Waiting for close lock at 1731014019161Disabling compacts and flushes for region at 1731014019161Disabling writes for close at 1731014019161Writing region close event to WAL at 1731014019161Closed at 1731014019161 2024-11-07T21:13:39,162 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 534ea0d1ebd5ba65a9e4b728e47c1c2b, NAME => 'test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741847_1023 (size=45) 2024-11-07T21:13:39,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741847_1023 (size=45) 2024-11-07T21:13:39,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c9fb04694e7a790ffddb7dcec6e3022d, disabling compactions & flushes 2024-11-07T21:13:39,165 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:39,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:39,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. after waiting 0 ms 2024-11-07T21:13:39,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:39,165 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:39,166 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c9fb04694e7a790ffddb7dcec6e3022d: Waiting for close lock at 1731014019165Disabling compacts and flushes for region at 1731014019165Disabling writes for close at 1731014019165Writing region close event to WAL at 1731014019165Closed at 1731014019165 2024-11-07T21:13:39,166 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a0940719af3866502facc6a87d8e2e16, NAME => 'test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741848_1024 (size=45) 2024-11-07T21:13:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741848_1024 (size=45) 2024-11-07T21:13:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741848_1024 (size=45) 2024-11-07T21:13:39,183 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,183 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5dbe14ba56226cb0a73d7116364c76a7, disabling compactions & flushes 2024-11-07T21:13:39,183 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:39,183 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:39,183 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. after waiting 0 ms 2024-11-07T21:13:39,183 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:39,183 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:39,184 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5dbe14ba56226cb0a73d7116364c76a7: Waiting for close lock at 1731014019183Disabling compacts and flushes for region at 1731014019183Disabling writes for close at 1731014019183Writing region close event to WAL at 1731014019183Closed at 1731014019183 2024-11-07T21:13:39,185 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a03ddf51c24eac49d64e43449b12eac, NAME => 'test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741849_1025 (size=45) 2024-11-07T21:13:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741849_1025 (size=45) 2024-11-07T21:13:39,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741849_1025 (size=45) 2024-11-07T21:13:39,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,191 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 534ea0d1ebd5ba65a9e4b728e47c1c2b, disabling compactions & flushes 2024-11-07T21:13:39,191 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:39,191 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:39,191 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. after waiting 0 ms 2024-11-07T21:13:39,191 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:39,191 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:39,191 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 534ea0d1ebd5ba65a9e4b728e47c1c2b: Waiting for close lock at 1731014019190Disabling compacts and flushes for region at 1731014019190Disabling writes for close at 1731014019191 (+1 ms)Writing region close event to WAL at 1731014019191Closed at 1731014019191 2024-11-07T21:13:39,192 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 419d87fa67b075ba0bc43a42ebb17a10, NAME => 'test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741850_1026 (size=45) 2024-11-07T21:13:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741850_1026 (size=45) 2024-11-07T21:13:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741850_1026 (size=45) 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a0940719af3866502facc6a87d8e2e16, disabling compactions & flushes 2024-11-07T21:13:39,198 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. after waiting 0 ms 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:39,198 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:39,198 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a0940719af3866502facc6a87d8e2e16: Waiting for close lock at 1731014019198Disabling compacts and flushes for region at 1731014019198Disabling writes for close at 1731014019198Writing region close event to WAL at 1731014019198Closed at 1731014019198 2024-11-07T21:13:39,199 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c4e192a5a4e8acdabe498b5222169ffa, NAME => 'test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741851_1027 (size=45) 2024-11-07T21:13:39,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741851_1027 (size=45) 2024-11-07T21:13:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741851_1027 (size=45) 2024-11-07T21:13:39,208 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2a03ddf51c24eac49d64e43449b12eac, disabling compactions & flushes 2024-11-07T21:13:39,209 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:39,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:39,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. after waiting 0 ms 2024-11-07T21:13:39,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:39,209 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:39,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a03ddf51c24eac49d64e43449b12eac: Waiting for close lock at 1731014019209Disabling compacts and flushes for region at 1731014019209Disabling writes for close at 1731014019209Writing region close event to WAL at 1731014019209Closed at 1731014019209 2024-11-07T21:13:39,210 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3a9d116816ee1386f43dca8ec9afdd34, NAME => 'test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741852_1028 (size=45) 2024-11-07T21:13:39,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741852_1028 (size=45) 2024-11-07T21:13:39,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741852_1028 (size=45) 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 419d87fa67b075ba0bc43a42ebb17a10, disabling compactions & flushes 2024-11-07T21:13:39,215 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. after waiting 0 ms 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:39,215 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:39,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 419d87fa67b075ba0bc43a42ebb17a10: Waiting for close lock at 1731014019215Disabling compacts and flushes for region at 1731014019215Disabling writes for close at 1731014019215Writing region close event to WAL at 1731014019215Closed at 1731014019215 2024-11-07T21:13:39,216 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ee05ff1f29e1c039ede7475cad6aceec, NAME => 'test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741853_1029 (size=45) 2024-11-07T21:13:39,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741853_1029 (size=45) 2024-11-07T21:13:39,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741853_1029 (size=45) 2024-11-07T21:13:39,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c4e192a5a4e8acdabe498b5222169ffa, disabling compactions & flushes 2024-11-07T21:13:39,225 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:39,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:39,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. after waiting 0 ms 2024-11-07T21:13:39,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:39,225 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:39,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c4e192a5a4e8acdabe498b5222169ffa: Waiting for close lock at 1731014019225Disabling compacts and flushes for region at 1731014019225Disabling writes for close at 1731014019225Writing region close event to WAL at 1731014019225Closed at 1731014019225 2024-11-07T21:13:39,226 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3291e3fd115a397f9c83c70c468e38ad, NAME => 'test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741854_1030 (size=45) 2024-11-07T21:13:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741855_1031 (size=45) 2024-11-07T21:13:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741855_1031 (size=45) 2024-11-07T21:13:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741855_1031 (size=45) 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741854_1030 (size=45) 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ee05ff1f29e1c039ede7475cad6aceec, disabling compactions & flushes 2024-11-07T21:13:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741854_1030 (size=45) 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3a9d116816ee1386f43dca8ec9afdd34, disabling compactions & flushes 2024-11-07T21:13:39,235 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:39,235 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. after waiting 0 ms 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. after waiting 0 ms 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:39,235 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:39,235 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:39,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ee05ff1f29e1c039ede7475cad6aceec: Waiting for close lock at 1731014019235Disabling compacts and flushes for region at 1731014019235Disabling writes for close at 1731014019235Writing region close event to WAL at 1731014019235Closed at 1731014019235 2024-11-07T21:13:39,236 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3a9d116816ee1386f43dca8ec9afdd34: Waiting for close lock at 1731014019235Disabling compacts and flushes for region at 1731014019235Disabling writes for close at 1731014019235Writing region close event to WAL at 1731014019235Closed at 1731014019235 2024-11-07T21:13:39,236 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 97a30b4736a1c73009748656f9ecda05, NAME => 'test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,237 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88f57835b12668bd6ea98aef3bab0834, NAME => 'test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741856_1032 (size=45) 2024-11-07T21:13:39,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741856_1032 (size=45) 2024-11-07T21:13:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741856_1032 (size=45) 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3291e3fd115a397f9c83c70c468e38ad, disabling compactions & flushes 2024-11-07T21:13:39,243 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. after waiting 0 ms 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:39,243 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:39,243 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3291e3fd115a397f9c83c70c468e38ad: Waiting for close lock at 1731014019243Disabling compacts and flushes for region at 1731014019243Disabling writes for close at 1731014019243Writing region close event to WAL at 1731014019243Closed at 1731014019243 2024-11-07T21:13:39,244 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b49002ef10444e21498431b7581adcf8, NAME => 'test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741858_1034 (size=45) 2024-11-07T21:13:39,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741858_1034 (size=45) 2024-11-07T21:13:39,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741858_1034 (size=45) 2024-11-07T21:13:39,256 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,256 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 88f57835b12668bd6ea98aef3bab0834, disabling compactions & flushes 2024-11-07T21:13:39,256 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:39,256 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:39,256 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. after waiting 0 ms 2024-11-07T21:13:39,256 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:39,257 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:39,257 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88f57835b12668bd6ea98aef3bab0834: Waiting for close lock at 1731014019256Disabling compacts and flushes for region at 1731014019256Disabling writes for close at 1731014019256Writing region close event to WAL at 1731014019256Closed at 1731014019257 (+1 ms) 2024-11-07T21:13:39,258 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c7f4e6dd7021dd19751a806581e68a69, NAME => 'test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741857_1033 (size=45) 2024-11-07T21:13:39,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741857_1033 (size=45) 2024-11-07T21:13:39,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741857_1033 (size=45) 2024-11-07T21:13:39,259 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,260 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 97a30b4736a1c73009748656f9ecda05, disabling compactions & flushes 2024-11-07T21:13:39,260 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:39,260 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:39,260 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. after waiting 0 ms 2024-11-07T21:13:39,260 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:39,260 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:39,260 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 97a30b4736a1c73009748656f9ecda05: Waiting for close lock at 1731014019260Disabling compacts and flushes for region at 1731014019260Disabling writes for close at 1731014019260Writing region close event to WAL at 1731014019260Closed at 1731014019260 2024-11-07T21:13:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741859_1035 (size=45) 2024-11-07T21:13:39,261 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cbb358ace97aee494308e3aebf23cd6c, NAME => 'test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741859_1035 (size=45) 2024-11-07T21:13:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741859_1035 (size=45) 2024-11-07T21:13:39,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b49002ef10444e21498431b7581adcf8, disabling compactions & flushes 2024-11-07T21:13:39,262 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:39,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:39,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. after waiting 0 ms 2024-11-07T21:13:39,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:39,262 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:39,263 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b49002ef10444e21498431b7581adcf8: Waiting for close lock at 1731014019262Disabling compacts and flushes for region at 1731014019262Disabling writes for close at 1731014019262Writing region close event to WAL at 1731014019262Closed at 1731014019262 2024-11-07T21:13:39,263 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b79550b5deb80e15750d51a620ff5a34, NAME => 'test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741860_1036 (size=45) 2024-11-07T21:13:39,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741860_1036 (size=45) 2024-11-07T21:13:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741860_1036 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741862_1038 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741862_1038 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741862_1038 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741861_1037 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741861_1037 (size=45) 2024-11-07T21:13:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741861_1037 (size=45) 2024-11-07T21:13:39,282 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,282 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cbb358ace97aee494308e3aebf23cd6c, disabling compactions & flushes 2024-11-07T21:13:39,282 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:39,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:39,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. after waiting 0 ms 2024-11-07T21:13:39,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:39,283 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:39,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cbb358ace97aee494308e3aebf23cd6c: Waiting for close lock at 1731014019282Disabling compacts and flushes for region at 1731014019282Disabling writes for close at 1731014019283 (+1 ms)Writing region close event to WAL at 1731014019283Closed at 1731014019283 2024-11-07T21:13:39,284 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fb4ece95c1cd2dbd9a481f640546e011, NAME => 'test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af 2024-11-07T21:13:39,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741863_1039 (size=42) 2024-11-07T21:13:39,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741863_1039 (size=42) 2024-11-07T21:13:39,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741863_1039 (size=42) 2024-11-07T21:13:39,303 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fb4ece95c1cd2dbd9a481f640546e011, disabling compactions & flushes 2024-11-07T21:13:39,304 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:39,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:39,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. after waiting 0 ms 2024-11-07T21:13:39,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:39,304 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:39,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fb4ece95c1cd2dbd9a481f640546e011: Waiting for close lock at 1731014019304Disabling compacts and flushes for region at 1731014019304Disabling writes for close at 1731014019304Writing region close event to WAL at 1731014019304Closed at 1731014019304 2024-11-07T21:13:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:39,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c7f4e6dd7021dd19751a806581e68a69, disabling compactions & flushes 2024-11-07T21:13:39,675 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:39,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:39,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. after waiting 0 ms 2024-11-07T21:13:39,676 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:39,676 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:39,676 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c7f4e6dd7021dd19751a806581e68a69: Waiting for close lock at 1731014019675Disabling compacts and flushes for region at 1731014019675Disabling writes for close at 1731014019675Writing region close event to WAL at 1731014019676 (+1 ms)Closed at 1731014019676 2024-11-07T21:13:39,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:39,683 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b79550b5deb80e15750d51a620ff5a34, disabling compactions & flushes 2024-11-07T21:13:39,683 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:39,683 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:39,683 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. after waiting 0 ms 2024-11-07T21:13:39,683 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:39,683 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:39,683 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b79550b5deb80e15750d51a620ff5a34: Waiting for close lock at 1731014019683Disabling compacts and flushes for region at 1731014019683Disabling writes for close at 1731014019683Writing region close event to WAL at 1731014019683Closed at 1731014019683 2024-11-07T21:13:39,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T21:13:39,692 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,692 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731014019006.ca4a1386e839589539d875923c362933.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731014019686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731014019686"}]},"ts":"1731014019686"} 2024-11-07T21:13:39,739 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-07T21:13:39,741 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T21:13:39,744 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731014019742"}]},"ts":"1731014019742"} 2024-11-07T21:13:39,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-07T21:13:39,750 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {4fc742968c91=0} racks are {/default-rack=0} 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T21:13:39,766 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T21:13:39,766 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T21:13:39,766 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T21:13:39,766 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T21:13:39,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3136c80b02fddd6691fe6a2792b436c5, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=488c6da65f4aeee647de08a2f68cfcc9, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bdc220bbb3923bf71a5ec3496dfdf44b, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=028acc93496b53fbb3cfebe0328fb380, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17100f72b0290f4e0327d690648433ab, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca4a1386e839589539d875923c362933, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c9fb04694e7a790ffddb7dcec6e3022d, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=534ea0d1ebd5ba65a9e4b728e47c1c2b, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0940719af3866502facc6a87d8e2e16, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419d87fa67b075ba0bc43a42ebb17a10, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4e192a5a4e8acdabe498b5222169ffa, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9d116816ee1386f43dca8ec9afdd34, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3291e3fd115a397f9c83c70c468e38ad, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97a30b4736a1c73009748656f9ecda05, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b49002ef10444e21498431b7581adcf8, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7f4e6dd7021dd19751a806581e68a69, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb4ece95c1cd2dbd9a481f640546e011, ASSIGN}] 2024-11-07T21:13:39,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb4ece95c1cd2dbd9a481f640546e011, ASSIGN 2024-11-07T21:13:39,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fb4ece95c1cd2dbd9a481f640546e011, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:39,785 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:39,788 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,790 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7f4e6dd7021dd19751a806581e68a69, ASSIGN 2024-11-07T21:13:39,791 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c7f4e6dd7021dd19751a806581e68a69, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b49002ef10444e21498431b7581adcf8, ASSIGN 2024-11-07T21:13:39,794 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b49002ef10444e21498431b7581adcf8, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,796 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:39,797 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97a30b4736a1c73009748656f9ecda05, ASSIGN 2024-11-07T21:13:39,800 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97a30b4736a1c73009748656f9ecda05, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,801 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3291e3fd115a397f9c83c70c468e38ad, ASSIGN 2024-11-07T21:13:39,803 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3291e3fd115a397f9c83c70c468e38ad, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,804 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:39,805 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,807 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9d116816ee1386f43dca8ec9afdd34, ASSIGN 2024-11-07T21:13:39,808 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a9d116816ee1386f43dca8ec9afdd34, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,810 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4e192a5a4e8acdabe498b5222169ffa, ASSIGN 2024-11-07T21:13:39,811 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4e192a5a4e8acdabe498b5222169ffa, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419d87fa67b075ba0bc43a42ebb17a10, ASSIGN 2024-11-07T21:13:39,814 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=419d87fa67b075ba0bc43a42ebb17a10, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,815 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:39,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,818 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0940719af3866502facc6a87d8e2e16, ASSIGN 2024-11-07T21:13:39,820 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0940719af3866502facc6a87d8e2e16, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=534ea0d1ebd5ba65a9e4b728e47c1c2b, ASSIGN 2024-11-07T21:13:39,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=534ea0d1ebd5ba65a9e4b728e47c1c2b, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,824 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:39,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,826 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c9fb04694e7a790ffddb7dcec6e3022d, ASSIGN 2024-11-07T21:13:39,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c9fb04694e7a790ffddb7dcec6e3022d, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,829 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca4a1386e839589539d875923c362933, ASSIGN 2024-11-07T21:13:39,830 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ca4a1386e839589539d875923c362933, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,831 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:39,832 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17100f72b0290f4e0327d690648433ab, ASSIGN 2024-11-07T21:13:39,835 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17100f72b0290f4e0327d690648433ab, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=028acc93496b53fbb3cfebe0328fb380, ASSIGN 2024-11-07T21:13:39,838 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=028acc93496b53fbb3cfebe0328fb380, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,839 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:39,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,841 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bdc220bbb3923bf71a5ec3496dfdf44b, ASSIGN 2024-11-07T21:13:39,843 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bdc220bbb3923bf71a5ec3496dfdf44b, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,844 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=488c6da65f4aeee647de08a2f68cfcc9, ASSIGN 2024-11-07T21:13:39,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=488c6da65f4aeee647de08a2f68cfcc9, ASSIGN; state=OFFLINE, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:39,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3136c80b02fddd6691fe6a2792b436c5, ASSIGN 2024-11-07T21:13:39,848 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3136c80b02fddd6691fe6a2792b436c5, ASSIGN; state=OFFLINE, location=4fc742968c91,35389,1731014016411; forceNewPlan=false, retain=false 2024-11-07T21:13:39,850 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, ASSIGN 2024-11-07T21:13:39,852 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, ASSIGN; state=OFFLINE, location=4fc742968c91,38103,1731014016569; forceNewPlan=false, retain=false 2024-11-07T21:13:39,936 INFO [4fc742968c91:37625 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-07T21:13:39,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=97a30b4736a1c73009748656f9ecda05, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,941 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fb4ece95c1cd2dbd9a481f640546e011, ASSIGN because future has completed 2024-11-07T21:13:39,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=ca4a1386e839589539d875923c362933, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb4ece95c1cd2dbd9a481f640546e011, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97a30b4736a1c73009748656f9ecda05, ASSIGN because future has completed 2024-11-07T21:13:39,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a9d116816ee1386f43dca8ec9afdd34, ASSIGN because future has completed 2024-11-07T21:13:39,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=419d87fa67b075ba0bc43a42ebb17a10, ASSIGN because future has completed 2024-11-07T21:13:39,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=534ea0d1ebd5ba65a9e4b728e47c1c2b, ASSIGN because future has completed 2024-11-07T21:13:39,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 534ea0d1ebd5ba65a9e4b728e47c1c2b, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ca4a1386e839589539d875923c362933, ASSIGN because future has completed 2024-11-07T21:13:39,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca4a1386e839589539d875923c362933, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419d87fa67b075ba0bc43a42ebb17a10, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9d116816ee1386f43dca8ec9afdd34, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97a30b4736a1c73009748656f9ecda05, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=028acc93496b53fbb3cfebe0328fb380, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:39,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:39,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:39,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:39,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=028acc93496b53fbb3cfebe0328fb380, ASSIGN because future has completed 2024-11-07T21:13:39,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:39,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 028acc93496b53fbb3cfebe0328fb380, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:39,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=488c6da65f4aeee647de08a2f68cfcc9, ASSIGN because future has completed 2024-11-07T21:13:39,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN because future has completed 2024-11-07T21:13:39,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN because future has completed 2024-11-07T21:13:39,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN because future has completed 2024-11-07T21:13:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:39,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN because future has completed 2024-11-07T21:13:39,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488c6da65f4aeee647de08a2f68cfcc9, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:40,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPENING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c7f4e6dd7021dd19751a806581e68a69, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN because future has completed 2024-11-07T21:13:40,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN because future has completed 2024-11-07T21:13:40,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN because future has completed 2024-11-07T21:13:40,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN because future has completed 2024-11-07T21:13:40,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, ASSIGN because future has completed 2024-11-07T21:13:40,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c7f4e6dd7021dd19751a806581e68a69, ASSIGN because future has completed 2024-11-07T21:13:40,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7f4e6dd7021dd19751a806581e68a69, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:40,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b49002ef10444e21498431b7581adcf8, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,058 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=3291e3fd115a397f9c83c70c468e38ad, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a0940719af3866502facc6a87d8e2e16, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b49002ef10444e21498431b7581adcf8, ASSIGN because future has completed 2024-11-07T21:13:40,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=17100f72b0290f4e0327d690648433ab, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b49002ef10444e21498431b7581adcf8, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3291e3fd115a397f9c83c70c468e38ad, ASSIGN because future has completed 2024-11-07T21:13:40,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4e192a5a4e8acdabe498b5222169ffa, ASSIGN because future has completed 2024-11-07T21:13:40,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0940719af3866502facc6a87d8e2e16, ASSIGN because future has completed 2024-11-07T21:13:40,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c9fb04694e7a790ffddb7dcec6e3022d, ASSIGN because future has completed 2024-11-07T21:13:40,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb04694e7a790ffddb7dcec6e3022d, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17100f72b0290f4e0327d690648433ab, ASSIGN because future has completed 2024-11-07T21:13:40,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17100f72b0290f4e0327d690648433ab, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0940719af3866502facc6a87d8e2e16, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e192a5a4e8acdabe498b5222169ffa, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3291e3fd115a397f9c83c70c468e38ad, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3136c80b02fddd6691fe6a2792b436c5, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bdc220bbb3923bf71a5ec3496dfdf44b, ASSIGN because future has completed 2024-11-07T21:13:40,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc220bbb3923bf71a5ec3496dfdf44b, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3136c80b02fddd6691fe6a2792b436c5, ASSIGN because future has completed 2024-11-07T21:13:40,098 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T21:13:40,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3136c80b02fddd6691fe6a2792b436c5, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:40,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56961, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T21:13:40,107 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:40,107 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a9d116816ee1386f43dca8ec9afdd34, NAME => 'test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-07T21:13:40,107 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,107 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,107 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,108 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,110 INFO [StoreOpener-3a9d116816ee1386f43dca8ec9afdd34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,111 INFO [StoreOpener-3a9d116816ee1386f43dca8ec9afdd34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a9d116816ee1386f43dca8ec9afdd34 columnFamilyName family 2024-11-07T21:13:40,111 DEBUG [StoreOpener-3a9d116816ee1386f43dca8ec9afdd34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,112 INFO [StoreOpener-3a9d116816ee1386f43dca8ec9afdd34-1 {}] regionserver.HStore(327): Store=3a9d116816ee1386f43dca8ec9afdd34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,112 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,113 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,114 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,114 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,114 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,116 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,119 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3a9d116816ee1386f43dca8ec9afdd34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,120 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 3a9d116816ee1386f43dca8ec9afdd34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59757712, jitterRate=-0.10954070091247559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,120 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:40,122 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 3a9d116816ee1386f43dca8ec9afdd34: Running coprocessor pre-open hook at 1731014020108Writing region info on filesystem at 1731014020108Initializing all the Stores at 1731014020109 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020109Cleaning up temporary data from old regions at 1731014020114 (+5 ms)Running coprocessor post-open hooks at 1731014020120 (+6 ms)Region opened successfully at 1731014020121 (+1 ms) 2024-11-07T21:13:40,123 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34., pid=35, masterSystemTime=1731014020098 2024-11-07T21:13:40,126 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:40,126 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:40,126 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:40,127 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 97a30b4736a1c73009748656f9ecda05, NAME => 'test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-07T21:13:40,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,127 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,127 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,127 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,127 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9d116816ee1386f43dca8ec9afdd34, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,131 INFO [StoreOpener-97a30b4736a1c73009748656f9ecda05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,133 INFO [StoreOpener-97a30b4736a1c73009748656f9ecda05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a30b4736a1c73009748656f9ecda05 columnFamilyName family 2024-11-07T21:13:40,133 DEBUG [StoreOpener-97a30b4736a1c73009748656f9ecda05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,134 INFO [StoreOpener-97a30b4736a1c73009748656f9ecda05-1 {}] regionserver.HStore(327): Store=97a30b4736a1c73009748656f9ecda05/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,134 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,135 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=21 2024-11-07T21:13:40,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a9d116816ee1386f43dca8ec9afdd34, server=4fc742968c91,38729,1731014016510 in 165 msec 2024-11-07T21:13:40,136 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,136 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,137 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a9d116816ee1386f43dca8ec9afdd34, ASSIGN in 367 msec 2024-11-07T21:13:40,139 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,142 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/97a30b4736a1c73009748656f9ecda05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,143 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 97a30b4736a1c73009748656f9ecda05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75270876, jitterRate=0.1216234564781189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,143 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:40,143 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 97a30b4736a1c73009748656f9ecda05: Running coprocessor pre-open hook at 1731014020128Writing region info on filesystem at 1731014020128Initializing all the Stores at 1731014020129 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020130 (+1 ms)Cleaning up temporary data from old regions at 1731014020137 (+7 ms)Running coprocessor post-open hooks at 1731014020143 (+6 ms)Region opened successfully at 1731014020143 2024-11-07T21:13:40,144 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05., pid=36, masterSystemTime=1731014020098 2024-11-07T21:13:40,147 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:40,147 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:40,147 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:40,148 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 488c6da65f4aeee647de08a2f68cfcc9, NAME => 'test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-07T21:13:40,148 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,148 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,149 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,149 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,152 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:40,152 INFO [StoreOpener-488c6da65f4aeee647de08a2f68cfcc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,152 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 364a8945e5957c7750a8e0c67047b7a3, NAME => 'test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-07T21:13:40,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97a30b4736a1c73009748656f9ecda05, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,152 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,153 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,153 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,153 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,154 INFO [StoreOpener-488c6da65f4aeee647de08a2f68cfcc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 488c6da65f4aeee647de08a2f68cfcc9 columnFamilyName family 2024-11-07T21:13:40,154 DEBUG [StoreOpener-488c6da65f4aeee647de08a2f68cfcc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,155 INFO [StoreOpener-488c6da65f4aeee647de08a2f68cfcc9-1 {}] regionserver.HStore(327): Store=488c6da65f4aeee647de08a2f68cfcc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,155 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,155 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,157 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,157 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,158 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,158 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,159 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 364a8945e5957c7750a8e0c67047b7a3 columnFamilyName family 2024-11-07T21:13:40,159 DEBUG [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,159 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] regionserver.HStore(327): Store=364a8945e5957c7750a8e0c67047b7a3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,160 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,160 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,161 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,161 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=24 2024-11-07T21:13:40,163 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 97a30b4736a1c73009748656f9ecda05, server=4fc742968c91,38729,1731014016510 in 181 msec 2024-11-07T21:13:40,163 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,163 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/488c6da65f4aeee647de08a2f68cfcc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,164 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 488c6da65f4aeee647de08a2f68cfcc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68469719, jitterRate=0.020278319716453552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,165 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:40,165 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 488c6da65f4aeee647de08a2f68cfcc9: Running coprocessor pre-open hook at 1731014020149Writing region info on filesystem at 1731014020149Initializing all the Stores at 1731014020150 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020150Cleaning up temporary data from old regions at 1731014020158 (+8 ms)Running coprocessor post-open hooks at 1731014020165 (+7 ms)Region opened successfully at 1731014020165 2024-11-07T21:13:40,165 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,166 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9., pid=42, masterSystemTime=1731014020098 2024-11-07T21:13:40,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=97a30b4736a1c73009748656f9ecda05, ASSIGN in 394 msec 2024-11-07T21:13:40,168 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:40,169 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:40,169 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:40,169 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 534ea0d1ebd5ba65a9e4b728e47c1c2b, NAME => 'test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-07T21:13:40,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,169 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,170 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,170 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,170 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,171 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/364a8945e5957c7750a8e0c67047b7a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,171 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 364a8945e5957c7750a8e0c67047b7a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66391395, jitterRate=-0.010691121220588684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,172 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:40,172 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 364a8945e5957c7750a8e0c67047b7a3: Running coprocessor pre-open hook at 1731014020153Writing region info on filesystem at 1731014020153Initializing all the Stores at 1731014020154 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020154Cleaning up temporary data from old regions at 1731014020163 (+9 ms)Running coprocessor post-open hooks at 1731014020172 (+9 ms)Region opened successfully at 1731014020172 2024-11-07T21:13:40,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488c6da65f4aeee647de08a2f68cfcc9, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:40,174 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3., pid=44, masterSystemTime=1731014020144 2024-11-07T21:13:40,176 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:40,176 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:40,177 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:40,177 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e6e832a4cf4736e7c0513d0ad428d1, NAME => 'test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-07T21:13:40,177 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,177 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,178 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,178 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=7 2024-11-07T21:13:40,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 488c6da65f4aeee647de08a2f68cfcc9, server=4fc742968c91,38729,1731014016510 in 163 msec 2024-11-07T21:13:40,178 INFO [StoreOpener-534ea0d1ebd5ba65a9e4b728e47c1c2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,181 INFO [StoreOpener-534ea0d1ebd5ba65a9e4b728e47c1c2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 534ea0d1ebd5ba65a9e4b728e47c1c2b columnFamilyName family 2024-11-07T21:13:40,181 DEBUG [StoreOpener-534ea0d1ebd5ba65a9e4b728e47c1c2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=488c6da65f4aeee647de08a2f68cfcc9, ASSIGN in 409 msec 2024-11-07T21:13:40,182 INFO [StoreOpener-534ea0d1ebd5ba65a9e4b728e47c1c2b-1 {}] regionserver.HStore(327): Store=534ea0d1ebd5ba65a9e4b728e47c1c2b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,182 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,182 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,184 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=364a8945e5957c7750a8e0c67047b7a3, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,184 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,186 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,186 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,186 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e6e832a4cf4736e7c0513d0ad428d1 columnFamilyName family 2024-11-07T21:13:40,186 DEBUG [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,187 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] regionserver.HStore(327): Store=c1e6e832a4cf4736e7c0513d0ad428d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,187 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,188 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T21:13:40,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,189 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,189 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,190 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,190 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,190 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T21:13:40,197 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/534ea0d1ebd5ba65a9e4b728e47c1c2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=12 2024-11-07T21:13:40,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38103,1731014016569 in 163 msec 2024-11-07T21:13:40,197 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 534ea0d1ebd5ba65a9e4b728e47c1c2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59501164, jitterRate=-0.11336356401443481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:40,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 534ea0d1ebd5ba65a9e4b728e47c1c2b: Running coprocessor pre-open hook at 1731014020170Writing region info on filesystem at 1731014020170Initializing all the Stores at 1731014020172 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020172Cleaning up temporary data from old regions at 1731014020186 (+14 ms)Running coprocessor post-open hooks at 1731014020198 (+12 ms)Region opened successfully at 1731014020198 2024-11-07T21:13:40,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,199 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b., pid=32, masterSystemTime=1731014020098 2024-11-07T21:13:40,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN in 428 msec 2024-11-07T21:13:40,202 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:40,202 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:40,202 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:40,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => ca4a1386e839589539d875923c362933, NAME => 'test,iii,1731014019006.ca4a1386e839589539d875923c362933.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-07T21:13:40,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1731014019006.ca4a1386e839589539d875923c362933.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,205 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:40,206 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => c7f4e6dd7021dd19751a806581e68a69, NAME => 'test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-07T21:13:40,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 534ea0d1ebd5ba65a9e4b728e47c1c2b, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,206 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,206 INFO [StoreOpener-ca4a1386e839589539d875923c362933-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,206 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,206 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c1e6e832a4cf4736e7c0513d0ad428d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,207 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,207 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,208 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened c1e6e832a4cf4736e7c0513d0ad428d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71804302, jitterRate=0.06996747851371765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,209 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:40,209 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for c1e6e832a4cf4736e7c0513d0ad428d1: Running coprocessor pre-open hook at 1731014020178Writing region info on filesystem at 1731014020178Initializing all the Stores at 1731014020180 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020180Cleaning up temporary data from old regions at 1731014020190 (+10 ms)Running coprocessor post-open hooks at 1731014020209 (+19 ms)Region opened successfully at 1731014020209 2024-11-07T21:13:40,210 INFO [StoreOpener-ca4a1386e839589539d875923c362933-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca4a1386e839589539d875923c362933 columnFamilyName family 2024-11-07T21:13:40,210 DEBUG [StoreOpener-ca4a1386e839589539d875923c362933-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,211 INFO [StoreOpener-ca4a1386e839589539d875923c362933-1 {}] regionserver.HStore(327): Store=ca4a1386e839589539d875923c362933/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,211 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1., pid=47, masterSystemTime=1731014020144 2024-11-07T21:13:40,211 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-07T21:13:40,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 534ea0d1ebd5ba65a9e4b728e47c1c2b, server=4fc742968c91,38729,1731014016510 in 257 msec 2024-11-07T21:13:40,213 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:40,214 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:40,214 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:40,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dbe14ba56226cb0a73d7116364c76a7, NAME => 'test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-07T21:13:40,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,214 INFO [StoreOpener-c7f4e6dd7021dd19751a806581e68a69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,217 INFO [StoreOpener-c7f4e6dd7021dd19751a806581e68a69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7f4e6dd7021dd19751a806581e68a69 columnFamilyName family 2024-11-07T21:13:40,217 DEBUG [StoreOpener-c7f4e6dd7021dd19751a806581e68a69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=534ea0d1ebd5ba65a9e4b728e47c1c2b, ASSIGN in 443 msec 2024-11-07T21:13:40,218 INFO [StoreOpener-c7f4e6dd7021dd19751a806581e68a69-1 {}] regionserver.HStore(327): Store=c7f4e6dd7021dd19751a806581e68a69/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,218 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,218 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,219 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,220 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,222 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,222 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,226 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,228 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,230 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ca4a1386e839589539d875923c362933/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=9 2024-11-07T21:13:40,231 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c7f4e6dd7021dd19751a806581e68a69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,231 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened ca4a1386e839589539d875923c362933; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69207692, jitterRate=0.03127497434616089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,38103,1731014016569 in 180 msec 2024-11-07T21:13:40,231 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca4a1386e839589539d875923c362933 2024-11-07T21:13:40,231 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for ca4a1386e839589539d875923c362933: Running coprocessor pre-open hook at 1731014020204Writing region info on filesystem at 1731014020204Initializing all the Stores at 1731014020205 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020206 (+1 ms)Cleaning up temporary data from old regions at 1731014020214 (+8 ms)Running coprocessor post-open hooks at 1731014020231 (+17 ms)Region opened successfully at 1731014020231 2024-11-07T21:13:40,232 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened c7f4e6dd7021dd19751a806581e68a69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62050191, jitterRate=-0.0753801017999649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,232 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731014019006.ca4a1386e839589539d875923c362933., pid=33, masterSystemTime=1731014020098 2024-11-07T21:13:40,232 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:40,232 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for c7f4e6dd7021dd19751a806581e68a69: Running coprocessor pre-open hook at 1731014020207Writing region info on filesystem at 1731014020207Initializing all the Stores at 1731014020210 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020210Cleaning up temporary data from old regions at 1731014020222 (+12 ms)Running coprocessor post-open hooks at 1731014020232 (+10 ms)Region opened successfully at 1731014020232 2024-11-07T21:13:40,235 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69., pid=45, masterSystemTime=1731014020188 2024-11-07T21:13:40,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN in 462 msec 2024-11-07T21:13:40,235 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dbe14ba56226cb0a73d7116364c76a7 columnFamilyName family 2024-11-07T21:13:40,235 DEBUG [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,236 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:40,236 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:40,236 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:40,236 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] regionserver.HStore(327): Store=5dbe14ba56226cb0a73d7116364c76a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,237 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 419d87fa67b075ba0bc43a42ebb17a10, NAME => 'test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-07T21:13:40,237 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,237 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,237 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,239 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=ca4a1386e839589539d875923c362933, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,239 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,240 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:40,240 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:40,240 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:40,240 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 3291e3fd115a397f9c83c70c468e38ad, NAME => 'test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-07T21:13:40,240 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,243 INFO [StoreOpener-419d87fa67b075ba0bc43a42ebb17a10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca4a1386e839589539d875923c362933, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,244 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,245 INFO [StoreOpener-3291e3fd115a397f9c83c70c468e38ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,246 INFO [StoreOpener-419d87fa67b075ba0bc43a42ebb17a10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419d87fa67b075ba0bc43a42ebb17a10 columnFamilyName family 2024-11-07T21:13:40,247 DEBUG [StoreOpener-419d87fa67b075ba0bc43a42ebb17a10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,248 INFO [StoreOpener-419d87fa67b075ba0bc43a42ebb17a10-1 {}] regionserver.HStore(327): Store=419d87fa67b075ba0bc43a42ebb17a10/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,249 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,249 INFO [StoreOpener-3291e3fd115a397f9c83c70c468e38ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3291e3fd115a397f9c83c70c468e38ad columnFamilyName family 2024-11-07T21:13:40,249 DEBUG [StoreOpener-3291e3fd115a397f9c83c70c468e38ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7f4e6dd7021dd19751a806581e68a69, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,251 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,251 INFO [StoreOpener-3291e3fd115a397f9c83c70c468e38ad-1 {}] regionserver.HStore(327): Store=3291e3fd115a397f9c83c70c468e38ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,251 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,252 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,253 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,253 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,253 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/5dbe14ba56226cb0a73d7116364c76a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-07T21:13:40,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure ca4a1386e839589539d875923c362933, server=4fc742968c91,38729,1731014016510 in 293 msec 2024-11-07T21:13:40,254 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,254 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 5dbe14ba56226cb0a73d7116364c76a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71619199, jitterRate=0.06720922887325287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,254 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:40,254 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 5dbe14ba56226cb0a73d7116364c76a7: Running coprocessor pre-open hook at 1731014020215Writing region info on filesystem at 1731014020215Initializing all the Stores at 1731014020217 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020217Cleaning up temporary data from old regions at 1731014020241 (+24 ms)Running coprocessor post-open hooks at 1731014020254 (+13 ms)Region opened successfully at 1731014020254 2024-11-07T21:13:40,256 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7., pid=48, masterSystemTime=1731014020144 2024-11-07T21:13:40,256 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,257 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,257 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,257 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ca4a1386e839589539d875923c362933, ASSIGN in 485 msec 2024-11-07T21:13:40,259 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:40,259 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:40,259 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:40,260 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => ee05ff1f29e1c039ede7475cad6aceec, NAME => 'test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-07T21:13:40,260 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,260 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,260 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,260 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,262 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/419d87fa67b075ba0bc43a42ebb17a10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,263 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 419d87fa67b075ba0bc43a42ebb17a10; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65810747, jitterRate=-0.019343450665473938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,263 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:40,261 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,263 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 419d87fa67b075ba0bc43a42ebb17a10: Running coprocessor pre-open hook at 1731014020238Writing region info on filesystem at 1731014020238Initializing all the Stores at 1731014020239 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020239Cleaning up temporary data from old regions at 1731014020253 (+14 ms)Running coprocessor post-open hooks at 1731014020263 (+10 ms)Region opened successfully at 1731014020263 2024-11-07T21:13:40,264 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10., pid=34, masterSystemTime=1731014020098 2024-11-07T21:13:40,264 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,268 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee05ff1f29e1c039ede7475cad6aceec columnFamilyName family 2024-11-07T21:13:40,268 DEBUG [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,268 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:40,268 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:40,268 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 028acc93496b53fbb3cfebe0328fb380, NAME => 'test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,269 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] regionserver.HStore(327): Store=ee05ff1f29e1c039ede7475cad6aceec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,269 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,270 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,271 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=27 2024-11-07T21:13:40,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure c7f4e6dd7021dd19751a806581e68a69, server=4fc742968c91,35389,1731014016411 in 230 msec 2024-11-07T21:13:40,272 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,272 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,274 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c7f4e6dd7021dd19751a806581e68a69, ASSIGN in 502 msec 2024-11-07T21:13:40,277 INFO [StoreOpener-028acc93496b53fbb3cfebe0328fb380-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,278 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3291e3fd115a397f9c83c70c468e38ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,279 INFO [StoreOpener-028acc93496b53fbb3cfebe0328fb380-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 028acc93496b53fbb3cfebe0328fb380 columnFamilyName family 2024-11-07T21:13:40,279 DEBUG [StoreOpener-028acc93496b53fbb3cfebe0328fb380-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,280 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 3291e3fd115a397f9c83c70c468e38ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72670112, jitterRate=0.08286905288696289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,280 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:40,280 INFO [StoreOpener-028acc93496b53fbb3cfebe0328fb380-1 {}] regionserver.HStore(327): Store=028acc93496b53fbb3cfebe0328fb380/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,280 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 3291e3fd115a397f9c83c70c468e38ad: Running coprocessor pre-open hook at 1731014020241Writing region info on filesystem at 1731014020241Initializing all the Stores at 1731014020243 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020243Cleaning up temporary data from old regions at 1731014020257 (+14 ms)Running coprocessor post-open hooks at 1731014020280 (+23 ms)Region opened successfully at 1731014020280 2024-11-07T21:13:40,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419d87fa67b075ba0bc43a42ebb17a10, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,280 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,282 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,282 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad., pid=54, masterSystemTime=1731014020188 2024-11-07T21:13:40,283 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,284 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,285 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,287 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:40,287 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:40,287 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:40,287 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 17100f72b0290f4e0327d690648433ab, NAME => 'test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-07T21:13:40,287 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,287 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,288 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,288 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=15 2024-11-07T21:13:40,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,38103,1731014016569 in 232 msec 2024-11-07T21:13:40,289 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,290 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened ee05ff1f29e1c039ede7475cad6aceec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69595824, jitterRate=0.03705859184265137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,290 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:40,290 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for ee05ff1f29e1c039ede7475cad6aceec: Running coprocessor pre-open hook at 1731014020261Writing region info on filesystem at 1731014020261Initializing all the Stores at 1731014020262 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020262Cleaning up temporary data from old regions at 1731014020272 (+10 ms)Running coprocessor post-open hooks at 1731014020290 (+18 ms)Region opened successfully at 1731014020290 2024-11-07T21:13:40,290 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,291 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec., pid=39, masterSystemTime=1731014020144 2024-11-07T21:13:40,292 INFO [StoreOpener-17100f72b0290f4e0327d690648433ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN in 520 msec 2024-11-07T21:13:40,295 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:40,295 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:40,295 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:40,296 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ebed7942a177c6981a4264c1bfd63ce, NAME => 'test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T21:13:40,296 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/028acc93496b53fbb3cfebe0328fb380/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,296 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,296 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,297 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,297 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,297 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 028acc93496b53fbb3cfebe0328fb380; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72641922, jitterRate=0.08244898915290833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,297 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:40,297 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 028acc93496b53fbb3cfebe0328fb380: Running coprocessor pre-open hook at 1731014020270Writing region info on filesystem at 1731014020270Initializing all the Stores at 1731014020271 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020271Cleaning up temporary data from old regions at 1731014020285 (+14 ms)Running coprocessor post-open hooks at 1731014020297 (+12 ms)Region opened successfully at 1731014020297 2024-11-07T21:13:40,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,298 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380., pid=37, masterSystemTime=1731014020098 2024-11-07T21:13:40,301 INFO [StoreOpener-17100f72b0290f4e0327d690648433ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17100f72b0290f4e0327d690648433ab columnFamilyName family 2024-11-07T21:13:40,301 DEBUG [StoreOpener-17100f72b0290f4e0327d690648433ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,303 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:40,303 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:40,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,303 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:40,303 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => fb4ece95c1cd2dbd9a481f640546e011, NAME => 'test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-07T21:13:40,303 INFO [StoreOpener-17100f72b0290f4e0327d690648433ab-1 {}] regionserver.HStore(327): Store=17100f72b0290f4e0327d690648433ab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,305 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,306 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,307 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,307 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,308 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,309 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,309 INFO [StoreOpener-fb4ece95c1cd2dbd9a481f640546e011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,310 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ebed7942a177c6981a4264c1bfd63ce columnFamilyName family 2024-11-07T21:13:40,311 DEBUG [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,312 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] regionserver.HStore(327): Store=8ebed7942a177c6981a4264c1bfd63ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,313 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,314 INFO [StoreOpener-fb4ece95c1cd2dbd9a481f640546e011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb4ece95c1cd2dbd9a481f640546e011 columnFamilyName family 2024-11-07T21:13:40,314 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,314 DEBUG [StoreOpener-fb4ece95c1cd2dbd9a481f640546e011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,315 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,315 INFO [StoreOpener-fb4ece95c1cd2dbd9a481f640546e011-1 {}] regionserver.HStore(327): Store=fb4ece95c1cd2dbd9a481f640546e011/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,316 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,316 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/17100f72b0290f4e0327d690648433ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,316 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,316 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,317 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 17100f72b0290f4e0327d690648433ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61659722, jitterRate=-0.0811985433101654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,317 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,317 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:40,317 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 17100f72b0290f4e0327d690648433ab: Running coprocessor pre-open hook at 1731014020288Writing region info on filesystem at 1731014020288Initializing all the Stores at 1731014020292 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020292Cleaning up temporary data from old regions at 1731014020307 (+15 ms)Running coprocessor post-open hooks at 1731014020317 (+10 ms)Region opened successfully at 1731014020317 2024-11-07T21:13:40,319 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,320 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab., pid=51, masterSystemTime=1731014020188 2024-11-07T21:13:40,320 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,320 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,321 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3291e3fd115a397f9c83c70c468e38ad, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,324 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,326 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,328 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 8ebed7942a177c6981a4264c1bfd63ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72334722, jitterRate=0.07787135243415833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,328 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:40,328 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 8ebed7942a177c6981a4264c1bfd63ce: Running coprocessor pre-open hook at 1731014020297Writing region info on filesystem at 1731014020297Initializing all the Stores at 1731014020298 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020298Cleaning up temporary data from old regions at 1731014020316 (+18 ms)Running coprocessor post-open hooks at 1731014020328 (+12 ms)Region opened successfully at 1731014020328 2024-11-07T21:13:40,328 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38729,1731014016510, table=test, region=419d87fa67b075ba0bc43a42ebb17a10. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,329 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:40,329 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:40,329 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:40,330 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => a0940719af3866502facc6a87d8e2e16, NAME => 'test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-07T21:13:40,330 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,331 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,331 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,331 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=23 2024-11-07T21:13:40,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 3291e3fd115a397f9c83c70c468e38ad, server=4fc742968c91,35389,1731014016411 in 240 msec 2024-11-07T21:13:40,334 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce., pid=46, masterSystemTime=1731014020144 2024-11-07T21:13:40,337 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:40,337 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:40,337 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:40,337 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 88f57835b12668bd6ea98aef3bab0834, NAME => 'test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-07T21:13:40,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3291e3fd115a397f9c83c70c468e38ad, ASSIGN in 564 msec 2024-11-07T21:13:40,338 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,338 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,339 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,339 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,341 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38103,1731014016569, table=test, region=8ebed7942a177c6981a4264c1bfd63ce. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,341 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38103,1731014016569, table=test, region=ee05ff1f29e1c039ede7475cad6aceec. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=17100f72b0290f4e0327d690648433ab, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17100f72b0290f4e0327d690648433ab, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,349 INFO [StoreOpener-a0940719af3866502facc6a87d8e2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,350 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/fb4ece95c1cd2dbd9a481f640546e011/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,351 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened fb4ece95c1cd2dbd9a481f640546e011; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73649802, jitterRate=0.0974675714969635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,351 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:40,351 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for fb4ece95c1cd2dbd9a481f640546e011: Running coprocessor pre-open hook at 1731014020304Writing region info on filesystem at 1731014020304Initializing all the Stores at 1731014020306 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020306Cleaning up temporary data from old regions at 1731014020320 (+14 ms)Running coprocessor post-open hooks at 1731014020351 (+31 ms)Region opened successfully at 1731014020351 2024-11-07T21:13:40,353 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011., pid=31, masterSystemTime=1731014020098 2024-11-07T21:13:40,355 INFO [StoreOpener-a0940719af3866502facc6a87d8e2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0940719af3866502facc6a87d8e2e16 columnFamilyName family 2024-11-07T21:13:40,355 DEBUG [StoreOpener-a0940719af3866502facc6a87d8e2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=5 2024-11-07T21:13:40,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569 in 309 msec 2024-11-07T21:13:40,356 INFO [StoreOpener-a0940719af3866502facc6a87d8e2e16-1 {}] regionserver.HStore(327): Store=a0940719af3866502facc6a87d8e2e16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,357 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,357 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,358 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,358 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:40,358 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:40,359 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, ASSIGN in 586 msec 2024-11-07T21:13:40,359 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88f57835b12668bd6ea98aef3bab0834 columnFamilyName family 2024-11-07T21:13:40,360 DEBUG [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,360 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,361 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,361 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] regionserver.HStore(327): Store=88f57835b12668bd6ea98aef3bab0834/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,361 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,362 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,362 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,363 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,363 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,363 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,366 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb4ece95c1cd2dbd9a481f640546e011, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,368 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/a0940719af3866502facc6a87d8e2e16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,369 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened a0940719af3866502facc6a87d8e2e16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69513044, jitterRate=0.03582507371902466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,370 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:40,370 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for a0940719af3866502facc6a87d8e2e16: Running coprocessor pre-open hook at 1731014020331Writing region info on filesystem at 1731014020331Initializing all the Stores at 1731014020333 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020333Cleaning up temporary data from old regions at 1731014020361 (+28 ms)Running coprocessor post-open hooks at 1731014020370 (+9 ms)Region opened successfully at 1731014020370 2024-11-07T21:13:40,371 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16., pid=52, masterSystemTime=1731014020188 2024-11-07T21:13:40,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=11 2024-11-07T21:13:40,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 17100f72b0290f4e0327d690648433ab, server=4fc742968c91,35389,1731014016411 in 295 msec 2024-11-07T21:13:40,374 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/88f57835b12668bd6ea98aef3bab0834/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,374 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:40,374 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:40,374 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:40,374 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => b49002ef10444e21498431b7581adcf8, NAME => 'test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,375 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 88f57835b12668bd6ea98aef3bab0834; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66485207, jitterRate=-0.009293213486671448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 88f57835b12668bd6ea98aef3bab0834: Running coprocessor pre-open hook at 1731014020339Writing region info on filesystem at 1731014020339Initializing all the Stores at 1731014020340 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020340Cleaning up temporary data from old regions at 1731014020363 (+23 ms)Running coprocessor post-open hooks at 1731014020375 (+12 ms)Region opened successfully at 1731014020375 2024-11-07T21:13:40,375 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,376 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834., pid=38, masterSystemTime=1731014020144 2024-11-07T21:13:40,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,379 INFO [StoreOpener-b49002ef10444e21498431b7581adcf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,380 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:40,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0940719af3866502facc6a87d8e2e16, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,380 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:40,381 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:40,381 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a03ddf51c24eac49d64e43449b12eac, NAME => 'test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-07T21:13:40,381 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,381 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,382 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,382 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=88f57835b12668bd6ea98aef3bab0834, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,385 INFO [StoreOpener-b49002ef10444e21498431b7581adcf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b49002ef10444e21498431b7581adcf8 columnFamilyName family 2024-11-07T21:13:40,385 DEBUG [StoreOpener-b49002ef10444e21498431b7581adcf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,386 INFO [StoreOpener-b49002ef10444e21498431b7581adcf8-1 {}] regionserver.HStore(327): Store=b49002ef10444e21498431b7581adcf8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,387 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,388 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,388 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,389 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,390 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,390 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,391 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a03ddf51c24eac49d64e43449b12eac columnFamilyName family 2024-11-07T21:13:40,392 DEBUG [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,393 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] regionserver.HStore(327): Store=2a03ddf51c24eac49d64e43449b12eac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,393 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=25 2024-11-07T21:13:40,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38103,1731014016569 in 397 msec 2024-11-07T21:13:40,394 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,395 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,396 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,397 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,397 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN in 625 msec 2024-11-07T21:13:40,398 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b49002ef10444e21498431b7581adcf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,399 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened b49002ef10444e21498431b7581adcf8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73518935, jitterRate=0.09551750123500824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,400 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:40,400 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for b49002ef10444e21498431b7581adcf8: Running coprocessor pre-open hook at 1731014020375Writing region info on filesystem at 1731014020375Initializing all the Stores at 1731014020377 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020377Cleaning up temporary data from old regions at 1731014020390 (+13 ms)Running coprocessor post-open hooks at 1731014020400 (+10 ms)Region opened successfully at 1731014020400 2024-11-07T21:13:40,401 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8., pid=49, masterSystemTime=1731014020188 2024-11-07T21:13:40,401 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,405 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:40,405 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:40,406 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:40,406 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => c9fb04694e7a790ffddb7dcec6e3022d, NAME => 'test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-07T21:13:40,406 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,406 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,407 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,407 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,409 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/2a03ddf51c24eac49d64e43449b12eac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,409 INFO [StoreOpener-c9fb04694e7a790ffddb7dcec6e3022d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,410 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 2a03ddf51c24eac49d64e43449b12eac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75488538, jitterRate=0.12486687302589417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,410 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:40,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-07T21:13:40,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure a0940719af3866502facc6a87d8e2e16, server=4fc742968c91,35389,1731014016411 in 322 msec 2024-11-07T21:13:40,410 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 2a03ddf51c24eac49d64e43449b12eac: Running coprocessor pre-open hook at 1731014020382Writing region info on filesystem at 1731014020382Initializing all the Stores at 1731014020384 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020384Cleaning up temporary data from old regions at 1731014020397 (+13 ms)Running coprocessor post-open hooks at 1731014020410 (+13 ms)Region opened successfully at 1731014020410 2024-11-07T21:13:40,412 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac., pid=43, masterSystemTime=1731014020144 2024-11-07T21:13:40,414 INFO [StoreOpener-c9fb04694e7a790ffddb7dcec6e3022d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb04694e7a790ffddb7dcec6e3022d columnFamilyName family 2024-11-07T21:13:40,414 DEBUG [StoreOpener-c9fb04694e7a790ffddb7dcec6e3022d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0940719af3866502facc6a87d8e2e16, ASSIGN in 642 msec 2024-11-07T21:13:40,415 INFO [StoreOpener-c9fb04694e7a790ffddb7dcec6e3022d-1 {}] regionserver.HStore(327): Store=c9fb04694e7a790ffddb7dcec6e3022d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,416 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:40,416 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:40,416 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,416 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:40,416 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b79550b5deb80e15750d51a620ff5a34, NAME => 'test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T21:13:40,417 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,417 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,417 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,417 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,417 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,418 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=2a03ddf51c24eac49d64e43449b12eac, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,419 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,419 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,420 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,423 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b79550b5deb80e15750d51a620ff5a34 columnFamilyName family 2024-11-07T21:13:40,423 DEBUG [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,424 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] regionserver.HStore(327): Store=b79550b5deb80e15750d51a620ff5a34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,424 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b49002ef10444e21498431b7581adcf8, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,424 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,426 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,427 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,428 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,428 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,431 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b49002ef10444e21498431b7581adcf8, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,434 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,35389,1731014016411, table=test, region=b49002ef10444e21498431b7581adcf8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,435 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38729,1731014016510, table=test, region=419d87fa67b075ba0bc43a42ebb17a10. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,435 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38729,1731014016510, table=test, region=fb4ece95c1cd2dbd9a481f640546e011. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-11-07T21:13:40,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38103,1731014016569 in 411 msec 2024-11-07T21:13:40,439 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,440 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c9fb04694e7a790ffddb7dcec6e3022d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,441 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b79550b5deb80e15750d51a620ff5a34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65406518, jitterRate=-0.025366932153701782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,441 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:40,441 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b79550b5deb80e15750d51a620ff5a34: Running coprocessor pre-open hook at 1731014020417Writing region info on filesystem at 1731014020417Initializing all the Stores at 1731014020418 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020419 (+1 ms)Cleaning up temporary data from old regions at 1731014020428 (+9 ms)Running coprocessor post-open hooks at 1731014020441 (+13 ms)Region opened successfully at 1731014020441 2024-11-07T21:13:40,442 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened c9fb04694e7a790ffddb7dcec6e3022d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74241650, jitterRate=0.10628679394721985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,442 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:40,442 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34., pid=41, masterSystemTime=1731014020144 2024-11-07T21:13:40,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN in 670 msec 2024-11-07T21:13:40,443 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for c9fb04694e7a790ffddb7dcec6e3022d: Running coprocessor pre-open hook at 1731014020407Writing region info on filesystem at 1731014020407Initializing all the Stores at 1731014020408 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020408Cleaning up temporary data from old regions at 1731014020419 (+11 ms)Running coprocessor post-open hooks at 1731014020442 (+23 ms)Region opened successfully at 1731014020443 (+1 ms) 2024-11-07T21:13:40,444 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d., pid=50, masterSystemTime=1731014020188 2024-11-07T21:13:40,447 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38103,1731014016569, table=test, region=ee05ff1f29e1c039ede7475cad6aceec. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,448 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:40,448 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:40,449 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:40,449 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => cbb358ace97aee494308e3aebf23cd6c, NAME => 'test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-07T21:13:40,449 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,449 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=26 2024-11-07T21:13:40,450 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b49002ef10444e21498431b7581adcf8, server=4fc742968c91,35389,1731014016411 in 383 msec 2024-11-07T21:13:40,450 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,450 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:40,450 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:40,451 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:40,451 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => c4e192a5a4e8acdabe498b5222169ffa, NAME => 'test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-07T21:13:40,452 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,452 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,452 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,452 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,452 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b49002ef10444e21498431b7581adcf8, ASSIGN in 681 msec 2024-11-07T21:13:40,455 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb358ace97aee494308e3aebf23cd6c columnFamilyName family 2024-11-07T21:13:40,455 DEBUG [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,456 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,456 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] regionserver.HStore(327): Store=cbb358ace97aee494308e3aebf23cd6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,456 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,456 INFO [StoreOpener-c4e192a5a4e8acdabe498b5222169ffa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,458 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,459 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb04694e7a790ffddb7dcec6e3022d, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,460 INFO [StoreOpener-c4e192a5a4e8acdabe498b5222169ffa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e192a5a4e8acdabe498b5222169ffa columnFamilyName family 2024-11-07T21:13:40,460 DEBUG [StoreOpener-c4e192a5a4e8acdabe498b5222169ffa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,461 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,461 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b79550b5deb80e15750d51a620ff5a34, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,462 INFO [StoreOpener-c4e192a5a4e8acdabe498b5222169ffa-1 {}] regionserver.HStore(327): Store=c4e192a5a4e8acdabe498b5222169ffa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,462 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,464 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,464 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,465 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,466 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,466 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,469 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,470 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,470 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened cbb358ace97aee494308e3aebf23cd6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74243197, jitterRate=0.10630984604358673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,470 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:40,470 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for cbb358ace97aee494308e3aebf23cd6c: Running coprocessor pre-open hook at 1731014020450Writing region info on filesystem at 1731014020450Initializing all the Stores at 1731014020451 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020451Cleaning up temporary data from old regions at 1731014020461 (+10 ms)Running coprocessor post-open hooks at 1731014020470 (+9 ms)Region opened successfully at 1731014020470 2024-11-07T21:13:40,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=14 2024-11-07T21:13:40,471 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c., pid=40, masterSystemTime=1731014020144 2024-11-07T21:13:40,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure c9fb04694e7a790ffddb7dcec6e3022d, server=4fc742968c91,35389,1731014016411 in 398 msec 2024-11-07T21:13:40,474 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c4e192a5a4e8acdabe498b5222169ffa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,475 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:40,475 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened c4e192a5a4e8acdabe498b5222169ffa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66015580, jitterRate=-0.01629120111465454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,475 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:40,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c9fb04694e7a790ffddb7dcec6e3022d, ASSIGN in 703 msec 2024-11-07T21:13:40,475 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:40,475 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for c4e192a5a4e8acdabe498b5222169ffa: Running coprocessor pre-open hook at 1731014020452Writing region info on filesystem at 1731014020452Initializing all the Stores at 1731014020454 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020454Cleaning up temporary data from old regions at 1731014020466 (+12 ms)Running coprocessor post-open hooks at 1731014020475 (+9 ms)Region opened successfully at 1731014020475 2024-11-07T21:13:40,476 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa., pid=53, masterSystemTime=1731014020188 2024-11-07T21:13:40,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=cbb358ace97aee494308e3aebf23cd6c, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:40,479 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:40,480 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:40,480 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:40,480 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 3136c80b02fddd6691fe6a2792b436c5, NAME => 'test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-07T21:13:40,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:40,480 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,481 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,481 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,481 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,484 INFO [StoreOpener-3136c80b02fddd6691fe6a2792b436c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e192a5a4e8acdabe498b5222169ffa, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,487 INFO [StoreOpener-3136c80b02fddd6691fe6a2792b436c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3136c80b02fddd6691fe6a2792b436c5 columnFamilyName family 2024-11-07T21:13:40,488 DEBUG [StoreOpener-3136c80b02fddd6691fe6a2792b436c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,489 INFO [StoreOpener-3136c80b02fddd6691fe6a2792b436c5-1 {}] regionserver.HStore(327): Store=3136c80b02fddd6691fe6a2792b436c5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,490 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,491 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=28 2024-11-07T21:13:40,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,38103,1731014016569 in 484 msec 2024-11-07T21:13:40,492 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,492 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,492 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,494 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN in 723 msec 2024-11-07T21:13:40,498 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3136c80b02fddd6691fe6a2792b436c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,498 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 3136c80b02fddd6691fe6a2792b436c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71280219, jitterRate=0.06215803325176239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,499 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:40,499 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 3136c80b02fddd6691fe6a2792b436c5: Running coprocessor pre-open hook at 1731014020481Writing region info on filesystem at 1731014020481Initializing all the Stores at 1731014020483 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020483Cleaning up temporary data from old regions at 1731014020492 (+9 ms)Running coprocessor post-open hooks at 1731014020499 (+7 ms)Region opened successfully at 1731014020499 2024-11-07T21:13:40,500 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5., pid=56, masterSystemTime=1731014020188 2024-11-07T21:13:40,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=20 2024-11-07T21:13:40,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c4e192a5a4e8acdabe498b5222169ffa, server=4fc742968c91,35389,1731014016411 in 414 msec 2024-11-07T21:13:40,503 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:40,503 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:40,504 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:40,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,504 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => bdc220bbb3923bf71a5ec3496dfdf44b, NAME => 'test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-07T21:13:40,504 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,504 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:40,504 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,505 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3136c80b02fddd6691fe6a2792b436c5, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c4e192a5a4e8acdabe498b5222169ffa, ASSIGN in 738 msec 2024-11-07T21:13:40,512 INFO [StoreOpener-bdc220bbb3923bf71a5ec3496dfdf44b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,515 INFO [StoreOpener-bdc220bbb3923bf71a5ec3496dfdf44b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdc220bbb3923bf71a5ec3496dfdf44b columnFamilyName family 2024-11-07T21:13:40,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-07T21:13:40,515 DEBUG [StoreOpener-bdc220bbb3923bf71a5ec3496dfdf44b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:40,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 3136c80b02fddd6691fe6a2792b436c5, server=4fc742968c91,35389,1731014016411 in 411 msec 2024-11-07T21:13:40,516 INFO [StoreOpener-bdc220bbb3923bf71a5ec3496dfdf44b-1 {}] regionserver.HStore(327): Store=bdc220bbb3923bf71a5ec3496dfdf44b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:40,516 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,517 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,518 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,518 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,518 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3136c80b02fddd6691fe6a2792b436c5, ASSIGN in 746 msec 2024-11-07T21:13:40,521 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,524 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/bdc220bbb3923bf71a5ec3496dfdf44b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T21:13:40,525 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened bdc220bbb3923bf71a5ec3496dfdf44b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73352369, jitterRate=0.09303547441959381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:40,525 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:40,525 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for bdc220bbb3923bf71a5ec3496dfdf44b: Running coprocessor pre-open hook at 1731014020505Writing region info on filesystem at 1731014020505Initializing all the Stores at 1731014020507 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014020507Cleaning up temporary data from old regions at 1731014020518 (+11 ms)Running coprocessor post-open hooks at 1731014020525 (+7 ms)Region opened successfully at 1731014020525 2024-11-07T21:13:40,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=29 2024-11-07T21:13:40,526 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b., pid=55, masterSystemTime=1731014020188 2024-11-07T21:13:40,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38103,1731014016569 in 516 msec 2024-11-07T21:13:40,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN in 758 msec 2024-11-07T21:13:40,529 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:40,530 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:40,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:40,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc220bbb3923bf71a5ec3496dfdf44b, server=4fc742968c91,35389,1731014016411 because future has completed 2024-11-07T21:13:40,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17100f72b0290f4e0327d690648433ab, ASSIGN in 763 msec 2024-11-07T21:13:40,539 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38729,1731014016510, table=test, region=419d87fa67b075ba0bc43a42ebb17a10. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,539 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38729,1731014016510, table=test, region=fb4ece95c1cd2dbd9a481f640546e011. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-07T21:13:40,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bdc220bbb3923bf71a5ec3496dfdf44b, server=4fc742968c91,35389,1731014016411 in 444 msec 2024-11-07T21:13:40,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bdc220bbb3923bf71a5ec3496dfdf44b, ASSIGN in 773 msec 2024-11-07T21:13:40,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T21:13:40,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure fb4ece95c1cd2dbd9a481f640546e011, server=4fc742968c91,38729,1731014016510 in 602 msec 2024-11-07T21:13:40,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fb4ece95c1cd2dbd9a481f640546e011, ASSIGN in 778 msec 2024-11-07T21:13:40,552 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4fc742968c91,38103,1731014016569, table=test, region=ee05ff1f29e1c039ede7475cad6aceec. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T21:13:40,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=22 2024-11-07T21:13:40,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,38103,1731014016569 in 556 msec 2024-11-07T21:13:40,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN in 787 msec 2024-11-07T21:13:40,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, openSeqNum=2, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:40,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 028acc93496b53fbb3cfebe0328fb380, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:40,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=19 2024-11-07T21:13:40,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 419d87fa67b075ba0bc43a42ebb17a10, server=4fc742968c91,38729,1731014016510 in 601 msec 2024-11-07T21:13:40,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=419d87fa67b075ba0bc43a42ebb17a10, ASSIGN in 798 msec 2024-11-07T21:13:40,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=10 2024-11-07T21:13:40,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 028acc93496b53fbb3cfebe0328fb380, server=4fc742968c91,38729,1731014016510 in 586 msec 2024-11-07T21:13:40,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-07T21:13:40,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=028acc93496b53fbb3cfebe0328fb380, ASSIGN in 804 msec 2024-11-07T21:13:40,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T21:13:40,577 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731014020577"}]},"ts":"1731014020577"} 2024-11-07T21:13:40,580 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-07T21:13:40,581 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T21:13:40,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5620 sec 2024-11-07T21:13:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T21:13:41,187 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-07T21:13:41,187 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-07T21:13:41,189 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:41,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-07T21:13:41,209 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:41,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-07T21:13:41,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:41,215 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-07T21:13:41,236 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-07T21:13:41,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T21:13:41,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T21:13:41,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T21:13:41,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T21:13:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3136c80b02fddd6691fe6a2792b436c5 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bdc220bbb3923bf71a5ec3496dfdf44b changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17100f72b0290f4e0327d690648433ab changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c9fb04694e7a790ffddb7dcec6e3022d changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0940719af3866502facc6a87d8e2e16 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c4e192a5a4e8acdabe498b5222169ffa changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3291e3fd115a397f9c83c70c468e38ad changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b49002ef10444e21498431b7581adcf8 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c7f4e6dd7021dd19751a806581e68a69 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8ebed7942a177c6981a4264c1bfd63ce changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c1e6e832a4cf4736e7c0513d0ad428d1 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 364a8945e5957c7750a8e0c67047b7a3 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5dbe14ba56226cb0a73d7116364c76a7 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2a03ddf51c24eac49d64e43449b12eac changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee05ff1f29e1c039ede7475cad6aceec changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88f57835b12668bd6ea98aef3bab0834 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cbb358ace97aee494308e3aebf23cd6c changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b79550b5deb80e15750d51a620ff5a34 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 488c6da65f4aeee647de08a2f68cfcc9 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 028acc93496b53fbb3cfebe0328fb380 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ca4a1386e839589539d875923c362933 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 534ea0d1ebd5ba65a9e4b728e47c1c2b changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 419d87fa67b075ba0bc43a42ebb17a10 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a9d116816ee1386f43dca8ec9afdd34 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97a30b4736a1c73009748656f9ecda05 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fb4ece95c1cd2dbd9a481f640546e011 changed from -1.0 to 0.0, refreshing cache 2024-11-07T21:13:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-07T21:13:41,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-07T21:13:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-07T21:13:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(204): Hosts are {4fc742968c91=0} racks are {/default-rack=0} 2024-11-07T21:13:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-07T21:13:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-07T21:13:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-07T21:13:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T21:13:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T21:13:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T21:13:41,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T21:13:41,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T21:13:41,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T21:13:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-07T21:13:41,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-07T21:13:41,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-07T21:13:41,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 457 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-07T21:13:41,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-07T21:13:41,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster(2172): balance hri=8ebed7942a177c6981a4264c1bfd63ce, source=4fc742968c91,38103,1731014016569, destination=4fc742968c91,38729,1731014016510 2024-11-07T21:13:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:41,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:41,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=CLOSING, regionLocation=4fc742968c91,38103,1731014016569 2024-11-07T21:13:41,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE because future has completed 2024-11-07T21:13:41,798 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T21:13:41,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569}] 2024-11-07T21:13:41,956 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:41,956 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T21:13:41,957 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 8ebed7942a177c6981a4264c1bfd63ce, disabling compactions & flushes 2024-11-07T21:13:41,957 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:41,957 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:41,957 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. after waiting 0 ms 2024-11-07T21:13:41,957 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:41,963 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:41,966 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:41,966 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 8ebed7942a177c6981a4264c1bfd63ce: Waiting for close lock at 1731014021957Running coprocessor pre-close hooks at 1731014021957Disabling compacts and flushes for region at 1731014021957Disabling writes for close at 1731014021957Writing region close event to WAL at 1731014021958 (+1 ms)Running coprocessor post-close hooks at 1731014021964 (+6 ms)Closed at 1731014021965 (+1 ms) 2024-11-07T21:13:41,966 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 8ebed7942a177c6981a4264c1bfd63ce move to 4fc742968c91,38729,1731014016510 record at close sequenceid=2 2024-11-07T21:13:41,969 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:41,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=CLOSED 2024-11-07T21:13:41,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569 because future has completed 2024-11-07T21:13:41,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T21:13:41,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38103,1731014016569 in 174 msec 2024-11-07T21:13:41,975 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE; state=CLOSED, location=4fc742968c91,38729,1731014016510; forceNewPlan=false, retain=false 2024-11-07T21:13:42,126 INFO [4fc742968c91:37625 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T21:13:42,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:42,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE because future has completed 2024-11-07T21:13:42,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:42,288 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:42,288 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ebed7942a177c6981a4264c1bfd63ce, NAME => 'test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T21:13:42,289 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,289 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:42,289 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,289 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,291 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,293 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ebed7942a177c6981a4264c1bfd63ce columnFamilyName family 2024-11-07T21:13:42,293 DEBUG [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:42,294 INFO [StoreOpener-8ebed7942a177c6981a4264c1bfd63ce-1 {}] regionserver.HStore(327): Store=8ebed7942a177c6981a4264c1bfd63ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:42,295 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,296 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,298 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,299 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,299 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,303 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,305 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 8ebed7942a177c6981a4264c1bfd63ce; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70289348, jitterRate=0.04739290475845337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:42,305 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:42,306 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 8ebed7942a177c6981a4264c1bfd63ce: Running coprocessor pre-open hook at 1731014022289Writing region info on filesystem at 1731014022289Initializing all the Stores at 1731014022291 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014022291Cleaning up temporary data from old regions at 1731014022299 (+8 ms)Running coprocessor post-open hooks at 1731014022305 (+6 ms)Region opened successfully at 1731014022306 (+1 ms) 2024-11-07T21:13:42,307 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce., pid=59, masterSystemTime=1731014022282 2024-11-07T21:13:42,310 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:42,310 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:42,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, openSeqNum=5, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:42,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38729,1731014016510 because future has completed 2024-11-07T21:13:42,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-07T21:13:42,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ebed7942a177c6981a4264c1bfd63ce, server=4fc742968c91,38729,1731014016510 in 187 msec 2024-11-07T21:13:42,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE in 531 msec 2024-11-07T21:13:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-07T21:13:42,394 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:42,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-07T21:13:42,418 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=383, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=2417 2024-11-07T21:13:42,421 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-07T21:13:42,422 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-07T21:13:42,422 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 4fc742968c91,38103,1731014016569 2024-11-07T21:13:42,422 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 4fc742968c91,38103,1731014016569: Simulated kill ***** 2024-11-07T21:13:42,423 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-07T21:13:42,425 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-07T21:13:42,427 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-07T21:13:42,427 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-07T21:13:42,427 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-07T21:13:42,428 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 334246912 }, "NonHeapMemoryUsage": { "committed": 153812992, "init": 7667712, "max": -1, "used": 151523944 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91" } ] } 2024-11-07T21:13:42,431 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37625 {}] master.MasterRpcServices(700): 4fc742968c91,38103,1731014016569 reported a fatal error: ***** ABORTING region server 4fc742968c91,38103,1731014016569: Simulated kill ***** 2024-11-07T21:13:42,432 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4fc742968c91,38103,1731014016569' ***** 2024-11-07T21:13:42,433 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-07T21:13:42,433 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T21:13:42,433 INFO [RS:2;4fc742968c91:38103 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-07T21:13:42,433 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T21:13:42,433 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:42,433 INFO [RS:2;4fc742968c91:38103 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-07T21:13:42,433 INFO [RS:2;4fc742968c91:38103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T21:13:42,434 INFO [RS:2;4fc742968c91:38103 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4fc742968c91:38103. 2024-11-07T21:13:42,434 DEBUG [RS:2;4fc742968c91:38103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T21:13:42,438 DEBUG [RS:2;4fc742968c91:38103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741836_1012 (size=33698) 2024-11-07T21:13:42,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741836_1012 (size=33698) 2024-11-07T21:13:42,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741836_1012 (size=33698) 2024-11-07T21:13:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741835_1011 (size=2764) 2024-11-07T21:13:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741835_1011 (size=2764) 2024-11-07T21:13:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741835_1011 (size=2764) 2024-11-07T21:13:42,452 DEBUG [RS:2;4fc742968c91:38103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:42,452 INFO [RS:2;4fc742968c91:38103 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:42,452 INFO [RS:2;4fc742968c91:38103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38103 2024-11-07T21:13:42,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T21:13:42,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4fc742968c91,38103,1731014016569 2024-11-07T21:13:42,512 INFO [RS:2;4fc742968c91:38103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T21:13:42,513 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4fc742968c91,38103,1731014016569] 2024-11-07T21:13:42,528 INFO [regionserver/4fc742968c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:42,533 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4fc742968c91,38103,1731014016569 already deleted, retry=false 2024-11-07T21:13:42,533 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4fc742968c91,38103,1731014016569 on 4fc742968c91,37625,1731014015654 2024-11-07T21:13:42,537 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:42,542 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 4fc742968c91,38103,1731014016569 (carryingMeta=true) 4fc742968c91,38103,1731014016569/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6b61c744[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-07T21:13:42,542 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:42,543 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true, isMeta: true 2024-11-07T21:13:42,546 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting 2024-11-07T21:13:42,547 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4fc742968c91,38103,1731014016569 WAL count=1, meta=true 2024-11-07T21:13:42,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta}] 2024-11-07T21:13:42,557 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:42,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:42,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:42,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38103-0x1011725b3200003, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:42,623 INFO [RS:2;4fc742968c91:38103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T21:13:42,624 INFO [RS:2;4fc742968c91:38103 {}] regionserver.HRegionServer(1031): Exiting; stopping=4fc742968c91,38103,1731014016569; zookeeper connection closed. 2024-11-07T21:13:42,624 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38150ffe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38150ffe 2024-11-07T21:13:42,635 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T21:13:42,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-07T21:13:42,750 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, size=32.9 K (33698bytes) 2024-11-07T21:13:42,751 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-07T21:13:42,751 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-07T21:13:42,751 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta 2024-11-07T21:13:42,755 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta after 3ms 2024-11-07T21:13:42,763 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta took 13ms 2024-11-07T21:13:42,771 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-07T21:13:42,776 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp 2024-11-07T21:13:42,780 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp 2024-11-07T21:13:42,782 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta so closing down 2024-11-07T21:13:42,782 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T21:13:42,784 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T21:13:42,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741864_1040 (size=33457) 2024-11-07T21:13:42,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741864_1040 (size=33457) 2024-11-07T21:13:42,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741864_1040 (size=33457) 2024-11-07T21:13:42,793 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-07T21:13:42,796 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-07T21:13:42,800 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, size=32.9 K, length=33698, corrupted=false, cancelled=false 2024-11-07T21:13:42,800 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, journal: Splitting hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, size=32.9 K (33698bytes) at 1731014022750Creating recovered edits writer path=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp at 1731014022780 (+30 ms)Finishing writing output for hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta so closing down at 1731014022782 (+2 ms)3 split writer threads finished at 1731014022784 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1731014022794 (+10 ms)Rename recovered edits hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta.temp to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731014022796 (+2 ms)Processed 62 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, size=32.9 K, length=33698, corrupted=false, cancelled=false at 1731014022800 (+4 ms) 2024-11-07T21:13:42,801 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta 2024-11-07T21:13:42,802 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-07T21:13:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-07T21:13:42,811 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs 2024-11-07T21:13:42,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T21:13:42,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 in 251 msec 2024-11-07T21:13:42,815 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T21:13:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 in 266 msec 2024-11-07T21:13:42,820 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting dir is empty, no logs to split. 2024-11-07T21:13:42,820 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4fc742968c91,38103,1731014016569 WAL count=0, meta=true 2024-11-07T21:13:42,820 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 4fc742968c91,38103,1731014016569 WAL splitting is done? wals=0, meta=true 2024-11-07T21:13:42,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T21:13:42,828 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:42,829 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:42,926 INFO [regionserver/4fc742968c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T21:13:42,926 INFO [regionserver/4fc742968c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T21:13:42,980 DEBUG [4fc742968c91:37625 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T21:13:42,980 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(204): Hosts are {4fc742968c91=0} racks are {/default-rack=0} 2024-11-07T21:13:42,981 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T21:13:42,981 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T21:13:42,981 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T21:13:42,981 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T21:13:42,981 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T21:13:42,981 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T21:13:42,981 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T21:13:42,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:42,982 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPENING 2024-11-07T21:13:42,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:42,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:42,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:42,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:42,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:43,160 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T21:13:43,160 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T21:13:43,161 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T21:13:43,163 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fc742968c91%2C38729%2C1731014016510.meta, suffix=.meta, logDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510, archiveDir=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs, maxLogs=32 2024-11-07T21:13:43,181 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510/4fc742968c91%2C38729%2C1731014016510.meta.1731014023166.meta, exclude list is [], retry=0 2024-11-07T21:13:43,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44057,DS-2b9b3999-346f-42a4-a048-4dc08136d4e0,DISK] 2024-11-07T21:13:43,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42963,DS-1028e527-7199-40c5-8ad9-53e5043a22d0,DISK] 2024-11-07T21:13:43,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45187,DS-fd90e12f-75e7-4279-a49f-f65b0a3e9772,DISK] 2024-11-07T21:13:43,187 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38729,1731014016510/4fc742968c91%2C38729%2C1731014016510.meta.1731014023166.meta 2024-11-07T21:13:43,187 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44915:44915),(127.0.0.1/127.0.0.1:34923:34923),(127.0.0.1/127.0.0.1:37905:37905)] 2024-11-07T21:13:43,187 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T21:13:43,187 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T21:13:43,188 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T21:13:43,188 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T21:13:43,188 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T21:13:43,188 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:43,188 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T21:13:43,188 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T21:13:43,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T21:13:43,191 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T21:13:43,191 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:43,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:43,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T21:13:43,192 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T21:13:43,192 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:43,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:43,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T21:13:43,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T21:13:43,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:43,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:43,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T21:13:43,195 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T21:13:43,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:43,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T21:13:43,196 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T21:13:43,197 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:43,199 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740 2024-11-07T21:13:43,200 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-07T21:13:43,214 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-07T21:13:43,217 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.36 KB heapSize=50.61 KB 2024-11-07T21:13:43,290 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/d01838cd6f18484190897c26ee5adc45 is 138, key is test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5./info:regioninfo/1731014020504/Put/seqid=0 2024-11-07T21:13:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741866_1042 (size=36326) 2024-11-07T21:13:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741866_1042 (size=36326) 2024-11-07T21:13:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741866_1042 (size=36326) 2024-11-07T21:13:43,311 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.21 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/d01838cd6f18484190897c26ee5adc45 2024-11-07T21:13:43,386 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/ns/2b040ea688864a0d943823a03ae98f6d is 43, key is default/ns:d/1731014018713/Put/seqid=0 2024-11-07T21:13:43,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741867_1043 (size=5153) 2024-11-07T21:13:43,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741867_1043 (size=5153) 2024-11-07T21:13:43,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741867_1043 (size=5153) 2024-11-07T21:13:43,395 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/ns/2b040ea688864a0d943823a03ae98f6d 2024-11-07T21:13:43,419 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/table/96b00f1b7ccd420aa84ccab41ace814f is 40, key is test/table:state/1731014020577/Put/seqid=0 2024-11-07T21:13:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741868_1044 (size=5165) 2024-11-07T21:13:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741868_1044 (size=5165) 2024-11-07T21:13:43,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741868_1044 (size=5165) 2024-11-07T21:13:43,429 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/table/96b00f1b7ccd420aa84ccab41ace814f 2024-11-07T21:13:43,441 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/d01838cd6f18484190897c26ee5adc45 as hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/info/d01838cd6f18484190897c26ee5adc45 2024-11-07T21:13:43,450 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/info/d01838cd6f18484190897c26ee5adc45, entries=273, sequenceid=64, filesize=35.5 K 2024-11-07T21:13:43,453 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/ns/2b040ea688864a0d943823a03ae98f6d as hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/ns/2b040ea688864a0d943823a03ae98f6d 2024-11-07T21:13:43,462 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/ns/2b040ea688864a0d943823a03ae98f6d, entries=2, sequenceid=64, filesize=5.0 K 2024-11-07T21:13:43,463 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/table/96b00f1b7ccd420aa84ccab41ace814f as hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/table/96b00f1b7ccd420aa84ccab41ace814f 2024-11-07T21:13:43,472 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/table/96b00f1b7ccd420aa84ccab41ace814f, entries=2, sequenceid=64, filesize=5.0 K 2024-11-07T21:13:43,476 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.36 KB/30060, heapSize ~50.31 KB/51520, currentSize=0 B/0 for 1588230740 in 258ms, sequenceid=64, compaction requested=false; wal=null 2024-11-07T21:13:43,477 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T21:13:43,480 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-07T21:13:43,482 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T21:13:43,482 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T21:13:43,483 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T21:13:43,484 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T21:13:43,487 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-07T21:13:43,488 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62016937, jitterRate=-0.07587562501430511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T21:13:43,488 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T21:13:43,488 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731014023188Writing region info on filesystem at 1731014023188Initializing all the Stores at 1731014023189 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014023189Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014023190 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014023190Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731014023190Obtaining lock to block concurrent updates at 1731014023217 (+27 ms)Preparing flush snapshotting stores in 1588230740 at 1731014023217Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30060, getHeapSize=51760, getOffHeapSize=0, getCellsCount=277 at 1731014023220 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731014023220Flushing 1588230740/info: creating writer at 1731014023221 (+1 ms)Flushing 1588230740/info: appending metadata at 1731014023276 (+55 ms)Flushing 1588230740/info: closing flushed file at 1731014023278 (+2 ms)Flushing 1588230740/ns: creating writer at 1731014023363 (+85 ms)Flushing 1588230740/ns: appending metadata at 1731014023384 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731014023384Flushing 1588230740/table: creating writer at 1731014023403 (+19 ms)Flushing 1588230740/table: appending metadata at 1731014023418 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731014023418Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@653e82fc: reopening flushed file at 1731014023439 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5507acbb: reopening flushed file at 1731014023451 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538fcff6: reopening flushed file at 1731014023462 (+11 ms)Finished flush of dataSize ~29.36 KB/30060, heapSize ~50.31 KB/51520, currentSize=0 B/0 for 1588230740 in 258ms, sequenceid=64, compaction requested=false; wal=null at 1731014023476 (+14 ms)Cleaning up temporary data from old regions at 1731014023482 (+6 ms)Running coprocessor post-open hooks at 1731014023488 (+6 ms)Region opened successfully at 1731014023488 2024-11-07T21:13:43,489 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731014023149 2024-11-07T21:13:43,493 DEBUG [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T21:13:43,493 INFO [RS_OPEN_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T21:13:43,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,495 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:43,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:43,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:43,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:43,543 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:43,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:43,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:43,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T21:13:43,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 in 548 msec 2024-11-07T21:13:43,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-07T21:13:43,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 728 msec 2024-11-07T21:13:43,553 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 4fc742968c91,38103,1731014016569 had 9 regions 2024-11-07T21:13:43,554 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true, isMeta: false 2024-11-07T21:13:43,556 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4fc742968c91,38103,1731014016569 WAL count=1, meta=false 2024-11-07T21:13:43,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086}] 2024-11-07T21:13:43,558 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:43,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-07T21:13:43,728 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086, size=2.7 K (2764bytes) 2024-11-07T21:13:43,728 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 2024-11-07T21:13:43,728 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 after 0ms 2024-11-07T21:13:43,731 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 took 4ms 2024-11-07T21:13:43,735 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 364a8945e5957c7750a8e0c67047b7a3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,737 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c1e6e832a4cf4736e7c0513d0ad428d1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,738 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5dbe14ba56226cb0a73d7116364c76a7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,740 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ee05ff1f29e1c039ede7475cad6aceec: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,742 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8ebed7942a177c6981a4264c1bfd63ce: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 88f57835b12668bd6ea98aef3bab0834: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,746 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2a03ddf51c24eac49d64e43449b12eac: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,748 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b79550b5deb80e15750d51a620ff5a34: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,749 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for cbb358ace97aee494308e3aebf23cd6c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T21:13:43,749 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 so closing down 2024-11-07T21:13:43,750 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T21:13:43,750 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T21:13:43,750 INFO [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086, size=2.7 K, length=2764, corrupted=false, cancelled=false 2024-11-07T21:13:43,750 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086, journal: Splitting hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086, size=2.7 K (2764bytes) at 1731014023728Finishing writing output for hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 so closing down at 1731014023749 (+21 ms)3 split writer threads finished at 1731014023750 (+1 ms)Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086, size=2.7 K, length=2764, corrupted=false, cancelled=false at 1731014023750 2024-11-07T21:13:43,750 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 2024-11-07T21:13:43,750 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4fc742968c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-07T21:13:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37625 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-07T21:13:43,754 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting/4fc742968c91%2C38103%2C1731014016569.1731014018086 to hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/oldWALs 2024-11-07T21:13:43,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-07T21:13:43,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 in 197 msec 2024-11-07T21:13:43,759 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-07T21:13:43,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 in 203 msec 2024-11-07T21:13:43,763 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/WALs/4fc742968c91,38103,1731014016569-splitting dir is empty, no logs to split. 2024-11-07T21:13:43,763 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4fc742968c91,38103,1731014016569 WAL count=0, meta=false 2024-11-07T21:13:43,763 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 4fc742968c91,38103,1731014016569 WAL splitting is done? wals=0, meta=false 2024-11-07T21:13:43,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN}] 2024-11-07T21:13:43,766 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:43,767 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:43,768 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,769 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:43,770 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,770 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:43,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,772 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:43,772 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:43,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,774 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:43,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:43,776 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(204): Hosts are {4fc742968c91=0} racks are {/default-rack=0} 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T21:13:43,918 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T21:13:43,918 INFO [4fc742968c91:37625 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T21:13:43,918 DEBUG [4fc742968c91:37625 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T21:13:43,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:43,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:43,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:43,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, regionLocation=4fc742968c91,35389,1731014016411 2024-11-07T21:13:43,920 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4fc742968c91:38103 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:43,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, regionLocation=4fc742968c91,38729,1731014016510 2024-11-07T21:13:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T21:13:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 from cache 2024-11-07T21:13:43,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 4fc742968c91:38103 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,923 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4fc742968c91:38103 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:43,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38103,1731014016569, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4fc742968c91:38103 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:43,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 4fc742968c91:38103 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4fc742968c91/172.17.0.2:38103 2024-11-07T21:13:44,033 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T21:13:44,033 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fc742968c91,38729,1731014016510, seqNum=-1] 2024-11-07T21:13:44,033 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T21:13:44,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60505, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T21:13:44,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN because future has completed 2024-11-07T21:13:44,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:44,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN because future has completed 2024-11-07T21:13:44,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN because future has completed 2024-11-07T21:13:44,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN because future has completed 2024-11-07T21:13:44,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T21:13:44,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN because future has completed 2024-11-07T21:13:44,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:44,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN because future has completed 2024-11-07T21:13:44,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN because future has completed 2024-11-07T21:13:44,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN because future has completed 2024-11-07T21:13:44,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:44,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:44,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:44,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411}] 2024-11-07T21:13:44,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:44,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38729,1731014016510}] 2024-11-07T21:13:44,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-07T21:13:44,198 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:44,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => ee05ff1f29e1c039ede7475cad6aceec, NAME => 'test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-07T21:13:44,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:44,198 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,199 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,200 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,201 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee05ff1f29e1c039ede7475cad6aceec columnFamilyName family 2024-11-07T21:13:44,202 DEBUG [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:44,202 INFO [StoreOpener-ee05ff1f29e1c039ede7475cad6aceec-1 {}] regionserver.HStore(327): Store=ee05ff1f29e1c039ede7475cad6aceec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:44,202 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,203 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,205 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,205 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,205 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,207 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => b79550b5deb80e15750d51a620ff5a34, NAME => 'test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T21:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,209 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened ee05ff1f29e1c039ede7475cad6aceec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62468549, jitterRate=-0.06914608180522919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:44,209 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:44,209 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for ee05ff1f29e1c039ede7475cad6aceec: Running coprocessor pre-open hook at 1731014024199Writing region info on filesystem at 1731014024199Initializing all the Stores at 1731014024200 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014024200Cleaning up temporary data from old regions at 1731014024205 (+5 ms)Running coprocessor post-open hooks at 1731014024209 (+4 ms)Region opened successfully at 1731014024209 2024-11-07T21:13:44,209 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,211 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec., pid=80, masterSystemTime=1731014024192 2024-11-07T21:13:44,211 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b79550b5deb80e15750d51a620ff5a34 columnFamilyName family 2024-11-07T21:13:44,211 DEBUG [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:44,212 INFO [StoreOpener-b79550b5deb80e15750d51a620ff5a34-1 {}] regionserver.HStore(327): Store=b79550b5deb80e15750d51a620ff5a34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:44,212 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,213 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:44,214 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:44,214 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:44,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => cbb358ace97aee494308e3aebf23cd6c, NAME => 'test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-07T21:13:44,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,214 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:44,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,215 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,216 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,217 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,217 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,217 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-07T21:13:44,217 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb358ace97aee494308e3aebf23cd6c columnFamilyName family 2024-11-07T21:13:44,218 DEBUG [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:44,218 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,218 INFO [StoreOpener-cbb358ace97aee494308e3aebf23cd6c-1 {}] regionserver.HStore(327): Store=cbb358ace97aee494308e3aebf23cd6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:44,218 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,219 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,219 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,218 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,221 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,221 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened b79550b5deb80e15750d51a620ff5a34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63974798, jitterRate=-0.04670122265815735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:44,221 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:44,221 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for b79550b5deb80e15750d51a620ff5a34: Running coprocessor pre-open hook at 1731014024208Writing region info on filesystem at 1731014024208Initializing all the Stores at 1731014024209 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014024209Cleaning up temporary data from old regions at 1731014024217 (+8 ms)Running coprocessor post-open hooks at 1731014024221 (+4 ms)Region opened successfully at 1731014024221 2024-11-07T21:13:44,222 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,222 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,223 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34., pid=78, masterSystemTime=1731014024202 2024-11-07T21:13:44,225 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,227 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened cbb358ace97aee494308e3aebf23cd6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72593525, jitterRate=0.08172781765460968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:44,227 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:44,227 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for cbb358ace97aee494308e3aebf23cd6c: Running coprocessor pre-open hook at 1731014024215Writing region info on filesystem at 1731014024215Initializing all the Stores at 1731014024216 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014024216Cleaning up temporary data from old regions at 1731014024222 (+6 ms)Running coprocessor post-open hooks at 1731014024227 (+5 ms)Region opened successfully at 1731014024227 2024-11-07T21:13:44,228 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:44,228 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:44,228 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. 2024-11-07T21:13:44,228 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c., pid=75, masterSystemTime=1731014024192 2024-11-07T21:13:44,228 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 88f57835b12668bd6ea98aef3bab0834, NAME => 'test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-07T21:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,231 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:44,232 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:44,232 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. 2024-11-07T21:13:44,232 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dbe14ba56226cb0a73d7116364c76a7, NAME => 'test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-07T21:13:44,232 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,231 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-07T21:13:44,232 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:44,233 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,233 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,233 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,234 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,235 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88f57835b12668bd6ea98aef3bab0834 columnFamilyName family 2024-11-07T21:13:44,235 DEBUG [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:44,235 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,236 INFO [StoreOpener-88f57835b12668bd6ea98aef3bab0834-1 {}] regionserver.HStore(327): Store=88f57835b12668bd6ea98aef3bab0834/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:44,236 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,236 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dbe14ba56226cb0a73d7116364c76a7 columnFamilyName family 2024-11-07T21:13:44,236 DEBUG [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:44,236 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,237 INFO [StoreOpener-5dbe14ba56226cb0a73d7116364c76a7-1 {}] regionserver.HStore(327): Store=5dbe14ba56226cb0a73d7116364c76a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:44,237 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,238 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,239 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,239 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,239 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,240 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,240 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,240 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,240 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,240 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,241 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,241 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 88f57835b12668bd6ea98aef3bab0834; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65644101, jitterRate=-0.021826669573783875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:44,242 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88f57835b12668bd6ea98aef3bab0834 2024-11-07T21:13:44,242 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 88f57835b12668bd6ea98aef3bab0834: Running coprocessor pre-open hook at 1731014024229Writing region info on filesystem at 1731014024229Initializing all the Stores at 1731014024231 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014024231Cleaning up temporary data from old regions at 1731014024238 (+7 ms)Running coprocessor post-open hooks at 1731014024242 (+4 ms)Region opened successfully at 1731014024242 2024-11-07T21:13:44,243 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834., pid=77, masterSystemTime=1731014024202 2024-11-07T21:13:44,243 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,244 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,247 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 5dbe14ba56226cb0a73d7116364c76a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59785257, jitterRate=-0.10913024842739105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:44,247 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dbe14ba56226cb0a73d7116364c76a7 2024-11-07T21:13:44,247 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 5dbe14ba56226cb0a73d7116364c76a7: Running coprocessor pre-open hook at 1731014024233Writing region info on filesystem at 1731014024233Initializing all the Stores at 1731014024234 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014024234Cleaning up temporary data from old regions at 1731014024241 (+7 ms)Running coprocessor post-open hooks at 1731014024247 (+6 ms)Region opened successfully at 1731014024247 2024-11-07T21:13:44,250 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7., pid=76, masterSystemTime=1731014024192 2024-11-07T21:13:44,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,250 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,263 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,263 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a03ddf51c24eac49d64e43449b12eac, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,265 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-07T21:13:44,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,268 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,268 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,268 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "4fc742968c91" port: 35389 start_code: 1731014016411 } transition { transition_code: OPENED region_info { region_id: 1731014019006 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1731014016672 }; retry (#0) after 1008ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T21:13:44,268 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "4fc742968c91" port: 38729 start_code: 1731014016510 } transition { transition_code: OPENED region_info { region_id: 1731014019006 table_name { namespace: "default" qualifier: "test" } start_key: "uuu" end_key: "vvv" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731014016672 }; retry (#0) after 1001ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T21:13:44,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,314 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,315 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,315 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,315 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054315 2024-11-07T21:13:44,315 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,315 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,319 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,357 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,357 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084357 2024-11-07T21:13:44,357 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144357 2024-11-07T21:13:44,357 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-07T21:13:44,357 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,357 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,357 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,357 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,358 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,358 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,358 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,358 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,371 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,371 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 364a8945e5957c7750a8e0c67047b7a3, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-07T21:13:44,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,372 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,372 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054387 2024-11-07T21:13:44,387 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,387 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,388 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,389 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,389 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,389 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,404 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084404 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144404 2024-11-07T21:13:44,404 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,404 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,404 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,404 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,405 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,414 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,414 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,419 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-07T21:13:44,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,421 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,421 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,429 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054429 2024-11-07T21:13:44,430 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,430 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,431 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,433 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,443 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,443 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084443 2024-11-07T21:13:44,443 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144443 2024-11-07T21:13:44,444 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-07T21:13:44,444 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,444 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,444 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,444 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,444 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,445 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,450 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e6e832a4cf4736e7c0513d0ad428d1, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-07T21:13:44,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,459 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,460 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054471 2024-11-07T21:13:44,471 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,471 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,473 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,475 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,475 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,475 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,484 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,484 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084484 2024-11-07T21:13:44,484 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144484 2024-11-07T21:13:44,484 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-07T21:13:44,484 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,484 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,484 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,485 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,485 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,485 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,491 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b79550b5deb80e15750d51a620ff5a34, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-07T21:13:44,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,500 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,501 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,512 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,512 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054513 2024-11-07T21:13:44,513 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,513 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,525 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084525 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144525 2024-11-07T21:13:44,525 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,525 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,525 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,525 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,526 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,533 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,533 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88f57835b12668bd6ea98aef3bab0834, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,534 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T21:13:44,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,535 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,537 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,538 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,544 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,544 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,544 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054545 2024-11-07T21:13:44,545 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,545 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,548 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,555 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084555 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144555 2024-11-07T21:13:44,555 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,555 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,555 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,555 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,556 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,556 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dbe14ba56226cb0a73d7116364c76a7, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T21:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,565 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,577 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054577 2024-11-07T21:13:44,577 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,577 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,578 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,580 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,587 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084587 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144587 2024-11-07T21:13:44,587 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,587 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,587 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,587 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,588 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,596 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,596 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb358ace97aee494308e3aebf23cd6c, server=4fc742968c91,35389,1731014016411 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T21:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054609 2024-11-07T21:13:44,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,609 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,610 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,611 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,611 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,611 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,621 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084621 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144621 2024-11-07T21:13:44,621 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,621 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,621 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,621 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,622 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,622 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN 2024-11-07T21:13:44,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T21:13:44,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,629 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,629 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5dbe14ba56226cb0a73d7116364c76a7, ASSIGN to state=OFFLINE, location=null, table=test, region=5dbe14ba56226cb0a73d7116364c76a7 to restore RIT 2024-11-07T21:13:44,640 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054640 2024-11-07T21:13:44,640 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,640 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,642 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,643 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,651 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084651 2024-11-07T21:13:44,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144651 2024-11-07T21:13:44,651 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-07T21:13:44,651 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,651 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,651 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,652 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,652 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,652 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,660 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN 2024-11-07T21:13:44,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-07T21:13:44,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,663 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e6e832a4cf4736e7c0513d0ad428d1 to restore RIT 2024-11-07T21:13:44,671 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054671 2024-11-07T21:13:44,671 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,671 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,674 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,680 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084680 2024-11-07T21:13:44,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144680 2024-11-07T21:13:44,680 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,680 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,680 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,680 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,681 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,681 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,681 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,686 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN 2024-11-07T21:13:44,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T21:13:44,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,687 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a03ddf51c24eac49d64e43449b12eac, ASSIGN to state=OFFLINE, location=null, table=test, region=2a03ddf51c24eac49d64e43449b12eac to restore RIT 2024-11-07T21:13:44,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054724 2024-11-07T21:13:44,724 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,724 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,729 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,734 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084734 2024-11-07T21:13:44,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144734 2024-11-07T21:13:44,734 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-07T21:13:44,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,734 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,735 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,735 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,735 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,735 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN 2024-11-07T21:13:44,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T21:13:44,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,745 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,748 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,749 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,756 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,756 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,756 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,756 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,756 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee05ff1f29e1c039ede7475cad6aceec, ASSIGN to state=OFFLINE, location=null, table=test, region=ee05ff1f29e1c039ede7475cad6aceec to restore RIT 2024-11-07T21:13:44,757 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054757 2024-11-07T21:13:44,757 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,757 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,759 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,760 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,760 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,760 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,766 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,766 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084766 2024-11-07T21:13:44,766 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144766 2024-11-07T21:13:44,766 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,766 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,766 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,766 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,767 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,767 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,771 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,771 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN 2024-11-07T21:13:44,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T21:13:44,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,776 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,787 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,787 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,787 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,787 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=364a8945e5957c7750a8e0c67047b7a3, ASSIGN to state=OFFLINE, location=null, table=test, region=364a8945e5957c7750a8e0c67047b7a3 to restore RIT 2024-11-07T21:13:44,787 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054787 2024-11-07T21:13:44,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,787 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,789 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,790 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,790 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,791 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,796 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084797 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144797 2024-11-07T21:13:44,797 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,797 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,797 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,797 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,798 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,803 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,803 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,803 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN 2024-11-07T21:13:44,803 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T21:13:44,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,804 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,804 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,818 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,818 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,818 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b79550b5deb80e15750d51a620ff5a34, ASSIGN to state=OFFLINE, location=null, table=test, region=b79550b5deb80e15750d51a620ff5a34 to restore RIT 2024-11-07T21:13:44,818 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054818 2024-11-07T21:13:44,818 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,818 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,819 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,823 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,826 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,826 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084826 2024-11-07T21:13:44,826 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144826 2024-11-07T21:13:44,826 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T21:13:44,827 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,827 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,827 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,827 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,827 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,827 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,828 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN 2024-11-07T21:13:44,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T21:13:44,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,834 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,835 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,856 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,857 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88f57835b12668bd6ea98aef3bab0834, ASSIGN to state=OFFLINE, location=null, table=test, region=88f57835b12668bd6ea98aef3bab0834 to restore RIT 2024-11-07T21:13:44,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054870 2024-11-07T21:13:44,870 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,870 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,871 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,872 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,873 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,878 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084878 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144878 2024-11-07T21:13:44,878 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,878 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,878 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,878 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,879 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN 2024-11-07T21:13:44,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T21:13:44,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,885 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cbb358ace97aee494308e3aebf23cd6c, ASSIGN to state=OFFLINE, location=null, table=test, region=cbb358ace97aee494308e3aebf23cd6c to restore RIT 2024-11-07T21:13:44,903 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054903 2024-11-07T21:13:44,903 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,903 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,904 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,905 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,911 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,911 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084911 2024-11-07T21:13:44,911 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144911 2024-11-07T21:13:44,911 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T21:13:44,911 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,912 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,912 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,912 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,912 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,912 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,913 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,918 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T21:13:44,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,920 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:44,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:44,934 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014054934 2024-11-07T21:13:44,934 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:44,934 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,936 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:44,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:44,937 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:44,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:44,943 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:44,943 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014084943 2024-11-07T21:13:44,943 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014144943 2024-11-07T21:13:44,943 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T21:13:44,943 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:44,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:44,943 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:44,943 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:44,944 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:44,944 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:44,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:44,944 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:44,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:44,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:44,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:44,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:44,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:44,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.1731014018086, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:44,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T21:13:44,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:44,952 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:44,952 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:44,970 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:44,971 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,024 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055024 2024-11-07T21:13:45,024 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,024 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:45,026 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,034 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085034 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145034 2024-11-07T21:13:45,034 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,034 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,034 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:45,034 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:45,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:45,035 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:45,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:45,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:45,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:45,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:45,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:45,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T21:13:45,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:45,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:45,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,039 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T21:13:45,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:45,040 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:45,040 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:45,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055093 2024-11-07T21:13:45,093 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,093 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,094 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,096 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:45,096 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:45,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,096 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,105 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085105 2024-11-07T21:13:45,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145105 2024-11-07T21:13:45,105 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-07T21:13:45,106 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:45,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,106 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,106 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,106 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:45,106 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:45,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:45,106 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:45,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:45,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:45,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:45,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:45,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T21:13:45,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:45,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:45,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T21:13:45,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:45,112 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:45,112 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:45,124 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,124 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37625 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,145 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055145 2024-11-07T21:13:45,145 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,145 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,146 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:45,147 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,153 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,153 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085153 2024-11-07T21:13:45,153 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145153 2024-11-07T21:13:45,153 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T21:13:45,153 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:45,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,153 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,153 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,154 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:45,154 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:45,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:45,154 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:45,155 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:45,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:45,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:45,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T21:13:45,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:45,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:45,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-07T21:13:45,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:45,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:45,159 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:45,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,165 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055165 2024-11-07T21:13:45,166 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,166 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,167 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,168 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:45,168 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,169 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,174 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,174 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085174 2024-11-07T21:13:45,174 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145174 2024-11-07T21:13:45,174 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T21:13:45,174 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:45,174 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,174 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,174 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,174 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:45,174 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:45,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:45,175 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:45,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:45,177 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:45,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T21:13:45,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:45,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:45,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:45,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,179 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:45,179 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 4fc742968c91%2C38103%2C1731014016569.meta.1731014018497.meta, worker=4fc742968c91,38729,1731014016510 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T21:13:45,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T21:13:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:45,180 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:45,180 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:45,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055186 2024-11-07T21:13:45,186 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,186 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T21:13:45,189 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,193 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085193 2024-11-07T21:13:45,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145193 2024-11-07T21:13:45,193 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-07T21:13:45,193 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T21:13:45,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,193 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,193 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,193 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T21:13:45,193 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T21:13:45,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T21:13:45,194 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T21:13:45,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T21:13:45,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T21:13:45,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T21:13:45,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T21:13:45,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebed7942a177c6981a4264c1bfd63ce, REOPEN/MOVE 2024-11-07T21:13:45,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true 2024-11-07T21:13:45,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T21:13:45,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-07T21:13:45,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=65 2024-11-07T21:13:45,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4fc742968c91,38729,1731014016510, table=hbase:meta, region=1588230740 2024-11-07T21:13:45,198 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fc742968c91,38729,1731014016510, state=OPEN 2024-11-07T21:13:45,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T21:13:45,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,207 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T21:13:45,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731014055207 2024-11-07T21:13:45,207 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T21:13:45,207 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T21:13:45,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T21:13:45,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure ee05ff1f29e1c039ede7475cad6aceec, server=4fc742968c91,35389,1731014016411:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4fc742968c91,38103,1731014016569, splitWal=true, meta=true exec-time=2.6750 sec 2024-11-07T21:13:45,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebed7942a177c6981a4264c1bfd63ce, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=5 2024-11-07T21:13:45,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3136c80b02fddd6691fe6a2792b436c5, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488c6da65f4aeee647de08a2f68cfcc9, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bdc220bbb3923bf71a5ec3496dfdf44b, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e6e832a4cf4736e7c0513d0ad428d1, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=028acc93496b53fbb3cfebe0328fb380, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17100f72b0290f4e0327d690648433ab, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=364a8945e5957c7750a8e0c67047b7a3, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca4a1386e839589539d875923c362933, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c9fb04694e7a790ffddb7dcec6e3022d, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dbe14ba56226cb0a73d7116364c76a7, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=534ea0d1ebd5ba65a9e4b728e47c1c2b, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0940719af3866502facc6a87d8e2e16, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a03ddf51c24eac49d64e43449b12eac, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419d87fa67b075ba0bc43a42ebb17a10, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e192a5a4e8acdabe498b5222169ffa, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9d116816ee1386f43dca8ec9afdd34, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee05ff1f29e1c039ede7475cad6aceec, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3291e3fd115a397f9c83c70c468e38ad, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a30b4736a1c73009748656f9ecda05, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88f57835b12668bd6ea98aef3bab0834, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b49002ef10444e21498431b7581adcf8, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7f4e6dd7021dd19751a806581e68a69, regionState=OPEN, lastHost=4fc742968c91,35389,1731014016411, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb358ace97aee494308e3aebf23cd6c, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,35389,1731014016411, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b79550b5deb80e15750d51a620ff5a34, regionState=OPENING, lastHost=4fc742968c91,38103,1731014016569, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb4ece95c1cd2dbd9a481f640546e011, regionState=OPEN, lastHost=4fc742968c91,38729,1731014016510, regionLocation=4fc742968c91,38729,1731014016510, openSeqNum=2 2024-11-07T21:13:45,213 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T21:13:45,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731014085213 2024-11-07T21:13:45,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731014145213 2024-11-07T21:13:45,213 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-07T21:13:45,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=349 (was 383), OpenFileDescriptor=624 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 330) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2345 (was 2417) 2024-11-07T21:13:45,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T21:13:45,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T21:13:45,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T21:13:45,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T21:13:45,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T21:13:45,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=149936055, stopped=false 2024-11-07T21:13:45,229 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4fc742968c91,37625,1731014015654 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T21:13:45,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:45,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:45,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T21:13:45,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T21:13:45,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,239 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4fc742968c91,35389,1731014016411' ***** 2024-11-07T21:13:45,239 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T21:13:45,239 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T21:13:45,239 INFO [RS:0;4fc742968c91:35389 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T21:13:45,239 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T21:13:45,239 INFO [RS:0;4fc742968c91:35389 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T21:13:45,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:45,239 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for bdc220bbb3923bf71a5ec3496dfdf44b 2024-11-07T21:13:45,239 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4fc742968c91,38729,1731014016510' ***** 2024-11-07T21:13:45,239 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T21:13:45,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T21:13:45,240 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for 3136c80b02fddd6691fe6a2792b436c5 2024-11-07T21:13:45,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 534ea0d1ebd5ba65a9e4b728e47c1c2b 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for b49002ef10444e21498431b7581adcf8 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for c9fb04694e7a790ffddb7dcec6e3022d 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for 17100f72b0290f4e0327d690648433ab 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for a0940719af3866502facc6a87d8e2e16 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for c7f4e6dd7021dd19751a806581e68a69 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ebed7942a177c6981a4264c1bfd63ce 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for 3291e3fd115a397f9c83c70c468e38ad 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for ca4a1386e839589539d875923c362933 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for cbb358ace97aee494308e3aebf23cd6c 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 419d87fa67b075ba0bc43a42ebb17a10 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for c4e192a5a4e8acdabe498b5222169ffa 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 97a30b4736a1c73009748656f9ecda05 2024-11-07T21:13:45,240 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(3091): Received CLOSE for ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a9d116816ee1386f43dca8ec9afdd34 2024-11-07T21:13:45,240 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 028acc93496b53fbb3cfebe0328fb380 2024-11-07T21:13:45,241 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(959): stopping server 4fc742968c91,35389,1731014016411 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for b79550b5deb80e15750d51a620ff5a34 2024-11-07T21:13:45,240 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 534ea0d1ebd5ba65a9e4b728e47c1c2b, disabling compactions & flushes 2024-11-07T21:13:45,241 INFO [RS:0;4fc742968c91:35389 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 488c6da65f4aeee647de08a2f68cfcc9 2024-11-07T21:13:45,240 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bdc220bbb3923bf71a5ec3496dfdf44b, disabling compactions & flushes 2024-11-07T21:13:45,241 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:45,241 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(959): stopping server 4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,241 INFO [RS:0;4fc742968c91:35389 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4fc742968c91:35389. 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. after waiting 0 ms 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. after waiting 0 ms 2024-11-07T21:13:45,241 DEBUG [RS:0;4fc742968c91:35389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4fc742968c91:38729. 2024-11-07T21:13:45,241 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:45,241 DEBUG [RS:0;4fc742968c91:35389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,241 DEBUG [RS:1;4fc742968c91:38729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T21:13:45,241 DEBUG [RS:1;4fc742968c91:38729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,241 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T21:13:45,241 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T21:13:45,241 DEBUG [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1351): Waiting on 17100f72b0290f4e0327d690648433ab, 3136c80b02fddd6691fe6a2792b436c5, 3291e3fd115a397f9c83c70c468e38ad, 5dbe14ba56226cb0a73d7116364c76a7, a0940719af3866502facc6a87d8e2e16, b49002ef10444e21498431b7581adcf8, bdc220bbb3923bf71a5ec3496dfdf44b, c4e192a5a4e8acdabe498b5222169ffa, c7f4e6dd7021dd19751a806581e68a69, c9fb04694e7a790ffddb7dcec6e3022d, cbb358ace97aee494308e3aebf23cd6c, ee05ff1f29e1c039ede7475cad6aceec 2024-11-07T21:13:45,242 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-07T21:13:45,242 DEBUG [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1351): Waiting on 028acc93496b53fbb3cfebe0328fb380, 1588230740, 3a9d116816ee1386f43dca8ec9afdd34, 419d87fa67b075ba0bc43a42ebb17a10, 488c6da65f4aeee647de08a2f68cfcc9, 534ea0d1ebd5ba65a9e4b728e47c1c2b, 88f57835b12668bd6ea98aef3bab0834, 8ebed7942a177c6981a4264c1bfd63ce, 97a30b4736a1c73009748656f9ecda05, b79550b5deb80e15750d51a620ff5a34, ca4a1386e839589539d875923c362933, fb4ece95c1cd2dbd9a481f640546e011 2024-11-07T21:13:45,242 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T21:13:45,242 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T21:13:45,242 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T21:13:45,242 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T21:13:45,242 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T21:13:45,242 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-07T21:13:45,245 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/bdc220bbb3923bf71a5ec3496dfdf44b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,245 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/534ea0d1ebd5ba65a9e4b728e47c1c2b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,246 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 534ea0d1ebd5ba65a9e4b728e47c1c2b: Waiting for close lock at 1731014025240Running coprocessor pre-close hooks at 1731014025240Disabling compacts and flushes for region at 1731014025240Disabling writes for close at 1731014025241 (+1 ms)Writing region close event to WAL at 1731014025241Running coprocessor post-close hooks at 1731014025246 (+5 ms)Closed at 1731014025246 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731014019006.534ea0d1ebd5ba65a9e4b728e47c1c2b. 2024-11-07T21:13:45,246 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bdc220bbb3923bf71a5ec3496dfdf44b: Waiting for close lock at 1731014025240Running coprocessor pre-close hooks at 1731014025240Disabling compacts and flushes for region at 1731014025240Disabling writes for close at 1731014025241 (+1 ms)Writing region close event to WAL at 1731014025241Running coprocessor post-close hooks at 1731014025246 (+5 ms)Closed at 1731014025246 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731014019006.bdc220bbb3923bf71a5ec3496dfdf44b. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb4ece95c1cd2dbd9a481f640546e011, disabling compactions & flushes 2024-11-07T21:13:45,246 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3136c80b02fddd6691fe6a2792b436c5, disabling compactions & flushes 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:45,246 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. after waiting 0 ms 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. after waiting 0 ms 2024-11-07T21:13:45,246 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:45,251 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/fb4ece95c1cd2dbd9a481f640546e011/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,252 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:45,252 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb4ece95c1cd2dbd9a481f640546e011: Waiting for close lock at 1731014025246Running coprocessor pre-close hooks at 1731014025246Disabling compacts and flushes for region at 1731014025246Disabling writes for close at 1731014025246Writing region close event to WAL at 1731014025247 (+1 ms)Running coprocessor post-close hooks at 1731014025252 (+5 ms)Closed at 1731014025252 2024-11-07T21:13:45,252 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3136c80b02fddd6691fe6a2792b436c5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,252 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731014019006.fb4ece95c1cd2dbd9a481f640546e011. 2024-11-07T21:13:45,252 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ebed7942a177c6981a4264c1bfd63ce, disabling compactions & flushes 2024-11-07T21:13:45,252 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:45,252 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. after waiting 0 ms 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:45,253 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3136c80b02fddd6691fe6a2792b436c5: Waiting for close lock at 1731014025246Running coprocessor pre-close hooks at 1731014025246Disabling compacts and flushes for region at 1731014025246Disabling writes for close at 1731014025246Writing region close event to WAL at 1731014025247 (+1 ms)Running coprocessor post-close hooks at 1731014025253 (+6 ms)Closed at 1731014025253 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731014019006.3136c80b02fddd6691fe6a2792b436c5. 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b49002ef10444e21498431b7581adcf8, disabling compactions & flushes 2024-11-07T21:13:45,253 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. after waiting 0 ms 2024-11-07T21:13:45,253 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:45,255 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/7d80742b78734b39bd3638d232104418 is 138, key is test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1./info:regioninfo/1731014023920/Put/seqid=0 2024-11-07T21:13:45,261 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/8ebed7942a177c6981a4264c1bfd63ce/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T21:13:45,261 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ebed7942a177c6981a4264c1bfd63ce: Waiting for close lock at 1731014025252Running coprocessor pre-close hooks at 1731014025252Disabling compacts and flushes for region at 1731014025252Disabling writes for close at 1731014025253 (+1 ms)Writing region close event to WAL at 1731014025253Running coprocessor post-close hooks at 1731014025261 (+8 ms)Closed at 1731014025261 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731014019006.8ebed7942a177c6981a4264c1bfd63ce. 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca4a1386e839589539d875923c362933, disabling compactions & flushes 2024-11-07T21:13:45,262 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731014019006.ca4a1386e839589539d875923c362933. after waiting 0 ms 2024-11-07T21:13:45,262 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:45,265 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b49002ef10444e21498431b7581adcf8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,266 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b49002ef10444e21498431b7581adcf8: Waiting for close lock at 1731014025253Running coprocessor pre-close hooks at 1731014025253Disabling compacts and flushes for region at 1731014025253Disabling writes for close at 1731014025253Writing region close event to WAL at 1731014025254 (+1 ms)Running coprocessor post-close hooks at 1731014025265 (+11 ms)Closed at 1731014025266 (+1 ms) 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731014019006.b49002ef10444e21498431b7581adcf8. 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c9fb04694e7a790ffddb7dcec6e3022d, disabling compactions & flushes 2024-11-07T21:13:45,266 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. after waiting 0 ms 2024-11-07T21:13:45,266 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:45,270 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834., aborting... java.io.IOException: Failed to report opened region to master: test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,271 ERROR [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 4fc742968c91,38729,1731014016510: Failed to open region test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,uuu,1731014019006.88f57835b12668bd6ea98aef3bab0834. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,271 ERROR [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-07T21:13:45,271 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-07T21:13:45,272 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-07T21:13:45,274 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-07T21:13:45,275 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c9fb04694e7a790ffddb7dcec6e3022d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,275 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-07T21:13:45,276 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c9fb04694e7a790ffddb7dcec6e3022d: Waiting for close lock at 1731014025266Running coprocessor pre-close hooks at 1731014025266Disabling compacts and flushes for region at 1731014025266Disabling writes for close at 1731014025266Writing region close event to WAL at 1731014025269 (+3 ms)Running coprocessor post-close hooks at 1731014025276 (+7 ms)Closed at 1731014025276 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731014019006.c9fb04694e7a790ffddb7dcec6e3022d. 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17100f72b0290f4e0327d690648433ab, disabling compactions & flushes 2024-11-07T21:13:45,276 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. after waiting 0 ms 2024-11-07T21:13:45,276 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:45,277 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7., aborting... java.io.IOException: Failed to report opened region to master: test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,277 ERROR [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2440): ***** ABORTING region server 4fc742968c91,35389,1731014016411: Failed to open region test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,kkk,1731014019006.5dbe14ba56226cb0a73d7116364c76a7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,277 ERROR [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-07T21:13:45,278 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-07T21:13:45,278 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-07T21:13:45,281 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-07T21:13:45,282 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-07T21:13:45,284 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 493630464 }, "NonHeapMemoryUsage": { "committed": 163512320, "init": 7667712, "max": -1, "used": 160867064 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38587, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 113, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 113, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 113, "QueueCallTime_min": 0, "QueueCallTime_max": 11, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 8, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 10, "QueueCallTime_98th_percentile": 10, "QueueCallTime_99th_percentile": 10, "QueueCallTime_99.9th_percentile": 10, "QueueCallTime_TimeRangeCount_0-1": 113, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 113, "TotalCallTime_min": 0, "TotalCallTime_max": 19, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 9, "TotalCallTime_75th_percentile": 14, "TotalCallTime_90th_percentile": 17, "TotalCallTime_95th_percentile": 18, "TotalCallTime_98th_percentile": 18, "TotalCallTime_99th_percentile": 18, "TotalCallTime_99.9th_percentile": 18, "TotalCallTime_TimeRangeCount_0-1": 113, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 113, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 605, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 113, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 113, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 113, "sentBytes": 54877 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62874", "tag.serverName": "4fc742968c91,35389,1731014016411", "tag.clusterId": "bf617550-7f62-4778-941c-0e640e550d51", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731014016411, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3333, "localBytesRead": 3333, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7458, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-07T21:13:45,285 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ca4a1386e839589539d875923c362933/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,286 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:45,286 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca4a1386e839589539d875923c362933: Waiting for close lock at 1731014025262Running coprocessor pre-close hooks at 1731014025262Disabling compacts and flushes for region at 1731014025262Disabling writes for close at 1731014025262Writing region close event to WAL at 1731014025268 (+6 ms)Running coprocessor post-close hooks at 1731014025286 (+18 ms)Closed at 1731014025286 2024-11-07T21:13:45,287 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731014019006.ca4a1386e839589539d875923c362933. 2024-11-07T21:13:45,287 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 419d87fa67b075ba0bc43a42ebb17a10, disabling compactions & flushes 2024-11-07T21:13:45,287 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:45,287 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:45,286 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4fc742968c91:37625 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4fc742968c91:37625 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-07T21:13:45,287 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. after waiting 0 ms 2024-11-07T21:13:45,287 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:45,291 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 495727616 }, "NonHeapMemoryUsage": { "committed": 163512320, "init": 7667712, "max": -1, "used": 160907840 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38587, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 113, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 113, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 113, "QueueCallTime_min": 0, "QueueCallTime_max": 11, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 8, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 10, "QueueCallTime_98th_percentile": 10, "QueueCallTime_99th_percentile": 10, "QueueCallTime_99.9th_percentile": 10, "QueueCallTime_TimeRangeCount_0-1": 113, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 113, "TotalCallTime_min": 0, "TotalCallTime_max": 19, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 9, "TotalCallTime_75th_percentile": 14, "TotalCallTime_90th_percentile": 17, "TotalCallTime_95th_percentile": 18, "TotalCallTime_98th_percentile": 18, "TotalCallTime_99th_percentile": 18, "TotalCallTime_99.9th_percentile": 18, "TotalCallTime_TimeRangeCount_0-1": 113, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 113, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 605, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 113, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 113, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 113, "sentBytes": 54877 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62874", "tag.serverName": "4fc742968c91,35389,1731014016411", "tag.clusterId": "bf617550-7f62-4778-941c-0e640e550d51", "tag.Context": "regionserver", "tag.Hostname": "4fc742968c91", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731014016411, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3333, "localBytesRead": 3333, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7458, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-07T21:13:45,291 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4fc742968c91:37625 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4fc742968c91:37625 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-07T21:13:45,292 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. 2024-11-07T21:13:45,292 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a03ddf51c24eac49d64e43449b12eac, NAME => 'test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-07T21:13:45,293 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,293 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:45,293 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,293 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741869_1045 (size=8195) 2024-11-07T21:13:45,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741869_1045 (size=8195) 2024-11-07T21:13:45,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741869_1045 (size=8195) 2024-11-07T21:13:45,298 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/7d80742b78734b39bd3638d232104418 2024-11-07T21:13:45,300 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. 2024-11-07T21:13:45,300 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e6e832a4cf4736e7c0513d0ad428d1, NAME => 'test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-07T21:13:45,301 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,301 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:45,301 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,301 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,302 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,303 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a03ddf51c24eac49d64e43449b12eac columnFamilyName family 2024-11-07T21:13:45,303 DEBUG [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:45,303 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/17100f72b0290f4e0327d690648433ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,304 INFO [StoreOpener-2a03ddf51c24eac49d64e43449b12eac-1 {}] regionserver.HStore(327): Store=2a03ddf51c24eac49d64e43449b12eac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:45,304 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,304 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:45,304 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17100f72b0290f4e0327d690648433ab: Waiting for close lock at 1731014025276Running coprocessor pre-close hooks at 1731014025276Disabling compacts and flushes for region at 1731014025276Disabling writes for close at 1731014025276Writing region close event to WAL at 1731014025287 (+11 ms)Running coprocessor post-close hooks at 1731014025304 (+17 ms)Closed at 1731014025304 2024-11-07T21:13:45,304 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731014019006.17100f72b0290f4e0327d690648433ab. 2024-11-07T21:13:45,305 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a0940719af3866502facc6a87d8e2e16, disabling compactions & flushes 2024-11-07T21:13:45,305 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:45,305 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:45,305 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. after waiting 0 ms 2024-11-07T21:13:45,305 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:45,305 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,307 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,308 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,308 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/.tmp/info/7d80742b78734b39bd3638d232104418 as hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/info/7d80742b78734b39bd3638d232104418 2024-11-07T21:13:45,308 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,308 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,309 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e6e832a4cf4736e7c0513d0ad428d1 columnFamilyName family 2024-11-07T21:13:45,309 DEBUG [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:45,309 INFO [StoreOpener-c1e6e832a4cf4736e7c0513d0ad428d1-1 {}] regionserver.HStore(327): Store=c1e6e832a4cf4736e7c0513d0ad428d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:45,310 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,310 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,310 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,311 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/419d87fa67b075ba0bc43a42ebb17a10/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,311 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 2a03ddf51c24eac49d64e43449b12eac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70953619, jitterRate=0.057291314005851746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:45,312 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a03ddf51c24eac49d64e43449b12eac 2024-11-07T21:13:45,312 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 2a03ddf51c24eac49d64e43449b12eac: Running coprocessor pre-open hook at 1731014025293Writing region info on filesystem at 1731014025293Initializing all the Stores at 1731014025294 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014025294Cleaning up temporary data from old regions at 1731014025308 (+14 ms)Running coprocessor post-open hooks at 1731014025312 (+4 ms)Region opened successfully at 1731014025312 2024-11-07T21:13:45,313 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 419d87fa67b075ba0bc43a42ebb17a10: Waiting for close lock at 1731014025287Running coprocessor pre-close hooks at 1731014025287Disabling compacts and flushes for region at 1731014025287Disabling writes for close at 1731014025287Writing region close event to WAL at 1731014025296 (+9 ms)Running coprocessor post-close hooks at 1731014025312 (+16 ms)Closed at 1731014025313 (+1 ms) 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731014019006.419d87fa67b075ba0bc43a42ebb17a10. 2024-11-07T21:13:45,313 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97a30b4736a1c73009748656f9ecda05, disabling compactions & flushes 2024-11-07T21:13:45,313 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. after waiting 0 ms 2024-11-07T21:13:45,313 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:45,313 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,314 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,314 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4fc742968c91,38729,1731014016510 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,314 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,nnn,1731014019006.2a03ddf51c24eac49d64e43449b12eac. and can not recover 2024-11-07T21:13:45,314 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. 2024-11-07T21:13:45,314 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 364a8945e5957c7750a8e0c67047b7a3, NAME => 'test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-07T21:13:45,315 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,315 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T21:13:45,315 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,315 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,317 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,318 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/info/7d80742b78734b39bd3638d232104418, entries=24, sequenceid=75, filesize=8.0 K 2024-11-07T21:13:45,319 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,319 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 77ms, sequenceid=75, compaction requested=false 2024-11-07T21:13:45,319 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 364a8945e5957c7750a8e0c67047b7a3 columnFamilyName family 2024-11-07T21:13:45,319 DEBUG [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T21:13:45,320 INFO [StoreOpener-364a8945e5957c7750a8e0c67047b7a3-1 {}] regionserver.HStore(327): Store=364a8945e5957c7750a8e0c67047b7a3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T21:13:45,320 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/a0940719af3866502facc6a87d8e2e16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,320 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,321 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened c1e6e832a4cf4736e7c0513d0ad428d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75195205, jitterRate=0.12049587070941925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:45,321 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1e6e832a4cf4736e7c0513d0ad428d1 2024-11-07T21:13:45,321 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for c1e6e832a4cf4736e7c0513d0ad428d1: Running coprocessor pre-open hook at 1731014025301Writing region info on filesystem at 1731014025301Initializing all the Stores at 1731014025303 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014025303Cleaning up temporary data from old regions at 1731014025314 (+11 ms)Running coprocessor post-open hooks at 1731014025321 (+7 ms)Region opened successfully at 1731014025321 2024-11-07T21:13:45,321 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,321 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/97a30b4736a1c73009748656f9ecda05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,321 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a0940719af3866502facc6a87d8e2e16: Waiting for close lock at 1731014025304Running coprocessor pre-close hooks at 1731014025304Disabling compacts and flushes for region at 1731014025304Disabling writes for close at 1731014025305 (+1 ms)Writing region close event to WAL at 1731014025312 (+7 ms)Running coprocessor post-close hooks at 1731014025321 (+9 ms)Closed at 1731014025321 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731014019006.a0940719af3866502facc6a87d8e2e16. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c7f4e6dd7021dd19751a806581e68a69, disabling compactions & flushes 2024-11-07T21:13:45,322 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:45,322 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. after waiting 0 ms 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97a30b4736a1c73009748656f9ecda05: Waiting for close lock at 1731014025313Running coprocessor pre-close hooks at 1731014025313Disabling compacts and flushes for region at 1731014025313Disabling writes for close at 1731014025313Writing region close event to WAL at 1731014025314 (+1 ms)Running coprocessor post-close hooks at 1731014025322 (+8 ms)Closed at 1731014025322 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731014019006.97a30b4736a1c73009748656f9ecda05. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a9d116816ee1386f43dca8ec9afdd34, disabling compactions & flushes 2024-11-07T21:13:45,322 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. after waiting 0 ms 2024-11-07T21:13:45,322 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:45,323 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,323 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,323 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4fc742968c91,35389,1731014016411 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,323 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,323 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,eee,1731014019006.c1e6e832a4cf4736e7c0513d0ad428d1. and can not recover 2024-11-07T21:13:45,326 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,327 INFO [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 364a8945e5957c7750a8e0c67047b7a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69162373, jitterRate=0.030599668622016907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T21:13:45,327 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 364a8945e5957c7750a8e0c67047b7a3 2024-11-07T21:13:45,327 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 364a8945e5957c7750a8e0c67047b7a3: Running coprocessor pre-open hook at 1731014025315Writing region info on filesystem at 1731014025315Initializing all the Stores at 1731014025317 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731014025317Cleaning up temporary data from old regions at 1731014025323 (+6 ms)Running coprocessor post-open hooks at 1731014025327 (+4 ms)Region opened successfully at 1731014025327 2024-11-07T21:13:45,328 WARN [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4fc742968c91,38729,1731014016510 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T21:13:45,329 DEBUG [RS_OPEN_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,hhh,1731014019006.364a8945e5957c7750a8e0c67047b7a3. and can not recover 2024-11-07T21:13:45,330 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c7f4e6dd7021dd19751a806581e68a69/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,330 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3a9d116816ee1386f43dca8ec9afdd34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,330 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:45,330 INFO [regionserver/4fc742968c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:45,330 INFO [regionserver/4fc742968c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:45,330 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c7f4e6dd7021dd19751a806581e68a69: Waiting for close lock at 1731014025322Running coprocessor pre-close hooks at 1731014025322Disabling compacts and flushes for region at 1731014025322Disabling writes for close at 1731014025322Writing region close event to WAL at 1731014025324 (+2 ms)Running coprocessor post-close hooks at 1731014025330 (+6 ms)Closed at 1731014025330 2024-11-07T21:13:45,331 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a9d116816ee1386f43dca8ec9afdd34: Waiting for close lock at 1731014025322Running coprocessor pre-close hooks at 1731014025322Disabling compacts and flushes for region at 1731014025322Disabling writes for close at 1731014025322Writing region close event to WAL at 1731014025325 (+3 ms)Running coprocessor post-close hooks at 1731014025330 (+5 ms)Closed at 1731014025330 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731014019006.c7f4e6dd7021dd19751a806581e68a69. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731014019006.3a9d116816ee1386f43dca8ec9afdd34. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 028acc93496b53fbb3cfebe0328fb380, disabling compactions & flushes 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3291e3fd115a397f9c83c70c468e38ad, disabling compactions & flushes 2024-11-07T21:13:45,331 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:45,331 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. after waiting 0 ms 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. after waiting 0 ms 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:45,331 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:45,332 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-07T21:13:45,333 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T21:13:45,333 INFO [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T21:13:45,333 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731014025242Running coprocessor pre-close hooks at 1731014025242Disabling compacts and flushes for region at 1731014025242Disabling writes for close at 1731014025242Obtaining lock to block concurrent updates at 1731014025242Preparing flush snapshotting stores in 1588230740 at 1731014025242Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731014025248 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731014025250 (+2 ms)Flushing 1588230740/info: creating writer at 1731014025250Flushing 1588230740/info: appending metadata at 1731014025254 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731014025254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f4988f: reopening flushed file at 1731014025307 (+53 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 77ms, sequenceid=75, compaction requested=false at 1731014025319 (+12 ms)Writing region close event to WAL at 1731014025326 (+7 ms)Running coprocessor post-close hooks at 1731014025333 (+7 ms)Closed at 1731014025333 2024-11-07T21:13:45,333 DEBUG [RS_CLOSE_META-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T21:13:45,335 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/028acc93496b53fbb3cfebe0328fb380/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,335 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/3291e3fd115a397f9c83c70c468e38ad/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,336 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:45,336 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 028acc93496b53fbb3cfebe0328fb380: Waiting for close lock at 1731014025331Running coprocessor pre-close hooks at 1731014025331Disabling compacts and flushes for region at 1731014025331Disabling writes for close at 1731014025331Writing region close event to WAL at 1731014025331Running coprocessor post-close hooks at 1731014025336 (+5 ms)Closed at 1731014025336 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3291e3fd115a397f9c83c70c468e38ad: Waiting for close lock at 1731014025331Running coprocessor pre-close hooks at 1731014025331Disabling compacts and flushes for region at 1731014025331Disabling writes for close at 1731014025331Writing region close event to WAL at 1731014025332 (+1 ms)Running coprocessor post-close hooks at 1731014025336 (+4 ms)Closed at 1731014025336 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731014019006.3291e3fd115a397f9c83c70c468e38ad. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731014019006.028acc93496b53fbb3cfebe0328fb380. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b79550b5deb80e15750d51a620ff5a34, disabling compactions & flushes 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cbb358ace97aee494308e3aebf23cd6c, disabling compactions & flushes 2024-11-07T21:13:45,336 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:45,336 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. after waiting 0 ms 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. after waiting 0 ms 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:45,336 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:45,340 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/b79550b5deb80e15750d51a620ff5a34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,340 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/cbb358ace97aee494308e3aebf23cd6c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,341 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b79550b5deb80e15750d51a620ff5a34: Waiting for close lock at 1731014025336Running coprocessor pre-close hooks at 1731014025336Disabling compacts and flushes for region at 1731014025336Disabling writes for close at 1731014025336Writing region close event to WAL at 1731014025337 (+1 ms)Running coprocessor post-close hooks at 1731014025341 (+4 ms)Closed at 1731014025341 2024-11-07T21:13:45,341 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cbb358ace97aee494308e3aebf23cd6c: Waiting for close lock at 1731014025336Running coprocessor pre-close hooks at 1731014025336Disabling compacts and flushes for region at 1731014025336Disabling writes for close at 1731014025336Writing region close event to WAL at 1731014025337 (+1 ms)Running coprocessor post-close hooks at 1731014025341 (+4 ms)Closed at 1731014025341 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731014019006.b79550b5deb80e15750d51a620ff5a34. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731014019006.cbb358ace97aee494308e3aebf23cd6c. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 488c6da65f4aeee647de08a2f68cfcc9, disabling compactions & flushes 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4e192a5a4e8acdabe498b5222169ffa, disabling compactions & flushes 2024-11-07T21:13:45,341 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:45,341 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. after waiting 0 ms 2024-11-07T21:13:45,341 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. after waiting 0 ms 2024-11-07T21:13:45,342 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:45,342 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:45,346 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/c4e192a5a4e8acdabe498b5222169ffa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,347 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4e192a5a4e8acdabe498b5222169ffa: Waiting for close lock at 1731014025341Running coprocessor pre-close hooks at 1731014025341Disabling compacts and flushes for region at 1731014025341Disabling writes for close at 1731014025342 (+1 ms)Writing region close event to WAL at 1731014025342Running coprocessor post-close hooks at 1731014025347 (+5 ms)Closed at 1731014025347 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731014019006.c4e192a5a4e8acdabe498b5222169ffa. 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ee05ff1f29e1c039ede7475cad6aceec, disabling compactions & flushes 2024-11-07T21:13:45,347 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. after waiting 0 ms 2024-11-07T21:13:45,347 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:45,350 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/488c6da65f4aeee647de08a2f68cfcc9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,351 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:45,351 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 488c6da65f4aeee647de08a2f68cfcc9: Waiting for close lock at 1731014025341Running coprocessor pre-close hooks at 1731014025341Disabling compacts and flushes for region at 1731014025341Disabling writes for close at 1731014025342 (+1 ms)Writing region close event to WAL at 1731014025342Running coprocessor post-close hooks at 1731014025351 (+9 ms)Closed at 1731014025351 2024-11-07T21:13:45,351 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731014019006.488c6da65f4aeee647de08a2f68cfcc9. 2024-11-07T21:13:45,354 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/data/default/test/ee05ff1f29e1c039ede7475cad6aceec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T21:13:45,355 INFO [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:45,355 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ee05ff1f29e1c039ede7475cad6aceec: Waiting for close lock at 1731014025347Running coprocessor pre-close hooks at 1731014025347Disabling compacts and flushes for region at 1731014025347Disabling writes for close at 1731014025347Writing region close event to WAL at 1731014025350 (+3 ms)Running coprocessor post-close hooks at 1731014025355 (+5 ms)Closed at 1731014025355 2024-11-07T21:13:45,355 DEBUG [RS_CLOSE_REGION-regionserver/4fc742968c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731014019006.ee05ff1f29e1c039ede7475cad6aceec. 2024-11-07T21:13:45,442 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(976): stopping server 4fc742968c91,35389,1731014016411; all regions closed. 2024-11-07T21:13:45,442 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(976): stopping server 4fc742968c91,38729,1731014016510; all regions closed. 2024-11-07T21:13:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741865_1041 (size=4204) 2024-11-07T21:13:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741833_1009 (size=6536) 2024-11-07T21:13:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741833_1009 (size=6536) 2024-11-07T21:13:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741865_1041 (size=4204) 2024-11-07T21:13:45,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741833_1009 (size=6536) 2024-11-07T21:13:45,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741865_1041 (size=4204) 2024-11-07T21:13:45,447 DEBUG [RS:0;4fc742968c91:35389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] hbase.ChoreService(370): Chore service for: regionserver/4fc742968c91:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T21:13:45,447 INFO [regionserver/4fc742968c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T21:13:45,447 INFO [RS:0;4fc742968c91:35389 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T21:13:45,448 INFO [RS:0;4fc742968c91:35389 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35389 2024-11-07T21:13:45,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741834_1010 (size=6257) 2024-11-07T21:13:45,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741834_1010 (size=6257) 2024-11-07T21:13:45,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741834_1010 (size=6257) 2024-11-07T21:13:45,450 DEBUG [RS:1;4fc742968c91:38729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T21:13:45,450 INFO [RS:1;4fc742968c91:38729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T21:13:45,450 INFO [RS:1;4fc742968c91:38729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T21:13:45,451 INFO [RS:1;4fc742968c91:38729 {}] hbase.ChoreService(370): Chore service for: regionserver/4fc742968c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T21:13:45,451 INFO [RS:1;4fc742968c91:38729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T21:13:45,451 INFO [regionserver/4fc742968c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T21:13:45,451 INFO [RS:1;4fc742968c91:38729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38729 2024-11-07T21:13:45,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4fc742968c91,35389,1731014016411 2024-11-07T21:13:45,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T21:13:45,459 INFO [RS:0;4fc742968c91:35389 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T21:13:45,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4fc742968c91,38729,1731014016510 2024-11-07T21:13:45,469 INFO [RS:1;4fc742968c91:38729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T21:13:45,470 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4fc742968c91,38729,1731014016510] 2024-11-07T21:13:45,490 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4fc742968c91,38729,1731014016510 already deleted, retry=false 2024-11-07T21:13:45,491 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4fc742968c91,38729,1731014016510 expired; onlineServers=1 2024-11-07T21:13:45,491 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4fc742968c91,35389,1731014016411] 2024-11-07T21:13:45,501 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4fc742968c91,35389,1731014016411 already deleted, retry=false 2024-11-07T21:13:45,501 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4fc742968c91,35389,1731014016411 expired; onlineServers=0 2024-11-07T21:13:45,501 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4fc742968c91,37625,1731014015654' ***** 2024-11-07T21:13:45,501 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T21:13:45,502 INFO [M:0;4fc742968c91:37625 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T21:13:45,502 INFO [M:0;4fc742968c91:37625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T21:13:45,502 DEBUG [M:0;4fc742968c91:37625 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T21:13:45,502 DEBUG [M:0;4fc742968c91:37625 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T21:13:45,502 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T21:13:45,502 DEBUG [master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.large.0-1731014017657 {}] cleaner.HFileCleaner(306): Exit Thread[master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.large.0-1731014017657,5,FailOnTimeoutGroup] 2024-11-07T21:13:45,502 DEBUG [master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.small.0-1731014017664 {}] cleaner.HFileCleaner(306): Exit Thread[master/4fc742968c91:0:becomeActiveMaster-HFileCleaner.small.0-1731014017664,5,FailOnTimeoutGroup] 2024-11-07T21:13:45,503 INFO [M:0;4fc742968c91:37625 {}] hbase.ChoreService(370): Chore service for: master/4fc742968c91:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T21:13:45,503 INFO [M:0;4fc742968c91:37625 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T21:13:45,503 DEBUG [M:0;4fc742968c91:37625 {}] master.HMaster(1795): Stopping service threads 2024-11-07T21:13:45,503 INFO [M:0;4fc742968c91:37625 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T21:13:45,504 INFO [M:0;4fc742968c91:37625 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T21:13:45,504 INFO [M:0;4fc742968c91:37625 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T21:13:45,504 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T21:13:45,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:45,580 INFO [RS:0;4fc742968c91:35389 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T21:13:45,580 INFO [RS:0;4fc742968c91:35389 {}] regionserver.HRegionServer(1031): Exiting; stopping=4fc742968c91,35389,1731014016411; zookeeper connection closed. 2024-11-07T21:13:45,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35389-0x1011725b3200001, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:45,581 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3d0b40ee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3d0b40ee 2024-11-07T21:13:45,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T21:13:45,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T21:13:45,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:45,591 INFO [RS:1;4fc742968c91:38729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T21:13:45,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x1011725b3200002, quorum=127.0.0.1:62874, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T21:13:45,591 INFO [RS:1;4fc742968c91:38729 {}] regionserver.HRegionServer(1031): Exiting; stopping=4fc742968c91,38729,1731014016510; zookeeper connection closed. 2024-11-07T21:13:45,591 DEBUG [M:0;4fc742968c91:37625 {}] zookeeper.ZKUtil(347): master:37625-0x1011725b3200000, quorum=127.0.0.1:62874, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T21:13:45,591 WARN [M:0;4fc742968c91:37625 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T21:13:45,592 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5fec09e8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5fec09e8 2024-11-07T21:13:45,593 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-07T21:13:45,593 INFO [M:0;4fc742968c91:37625 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/.lastflushedseqids 2024-11-07T21:13:45,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741870_1046 (size=1) 2024-11-07T21:13:45,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741870_1046 (size=1) 2024-11-07T21:13:45,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741870_1046 (size=1) 2024-11-07T21:13:45,609 INFO [M:0;4fc742968c91:37625 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T21:13:45,609 INFO [M:0;4fc742968c91:37625 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T21:13:45,609 DEBUG [M:0;4fc742968c91:37625 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T21:13:45,609 INFO [M:0;4fc742968c91:37625 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:45,609 DEBUG [M:0;4fc742968c91:37625 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:45,609 DEBUG [M:0;4fc742968c91:37625 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T21:13:45,609 DEBUG [M:0;4fc742968c91:37625 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T21:13:45,609 INFO [M:0;4fc742968c91:37625 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.10 KB heapSize=314.91 KB 2024-11-07T21:13:45,628 DEBUG [M:0;4fc742968c91:37625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21563d728df943a98dc116dc19060bdf is 82, key is hbase:meta,,1/info:regioninfo/1731014023494/Put/seqid=0 2024-11-07T21:13:45,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42963 is added to blk_1073741871_1047 (size=6071) 2024-11-07T21:13:45,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741871_1047 (size=6071) 2024-11-07T21:13:45,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44057 is added to blk_1073741871_1047 (size=6071) 2024-11-07T21:13:45,636 INFO [M:0;4fc742968c91:37625 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21563d728df943a98dc116dc19060bdf 2024-11-07T21:13:45,660 DEBUG [M:0;4fc742968c91:37625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42697/user/jenkins/test-data/2d3e6e99-6676-f81b-9848-c03484cb10af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c0f3b493c80544978808c1909bded879 is 2990, key is \x00\x00\x00\x00\x00\x00\x00