2024-11-09 21:13:20,135 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 21:13:20,147 main DEBUG Took 0.009887 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 21:13:20,147 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 21:13:20,148 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 21:13:20,149 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 21:13:20,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,157 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 21:13:20,170 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,172 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,173 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,175 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,175 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,177 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,178 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,178 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,179 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,180 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,181 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:13:20,182 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,182 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 21:13:20,183 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:13:20,185 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 21:13:20,186 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 21:13:20,187 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 21:13:20,188 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 21:13:20,188 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 21:13:20,197 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 21:13:20,200 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 21:13:20,202 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 21:13:20,202 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 21:13:20,202 main DEBUG createAppenders(={Console}) 2024-11-09 21:13:20,203 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-09 21:13:20,203 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 21:13:20,204 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-09 21:13:20,204 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 21:13:20,205 main DEBUG OutputStream closed 2024-11-09 21:13:20,205 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 21:13:20,205 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 21:13:20,205 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-09 21:13:20,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 21:13:20,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 21:13:20,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 21:13:20,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 21:13:20,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 21:13:20,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 21:13:20,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 21:13:20,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 21:13:20,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 21:13:20,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 21:13:20,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 21:13:20,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 21:13:20,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 21:13:20,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 21:13:20,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 21:13:20,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 21:13:20,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 21:13:20,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 21:13:20,286 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 21:13:20,286 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-09 21:13:20,287 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 21:13:20,287 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-09T21:13:20,300 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-09 21:13:20,303 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 21:13:20,303 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T21:13:20,544 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8 2024-11-09T21:13:20,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T21:13:20,582 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55, deleteOnExit=true 2024-11-09T21:13:20,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T21:13:20,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/test.cache.data in system properties and HBase conf 2024-11-09T21:13:20,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T21:13:20,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir in system properties and HBase conf 2024-11-09T21:13:20,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T21:13:20,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T21:13:20,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T21:13:20,671 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T21:13:20,764 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T21:13:20,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:13:20,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:13:20,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T21:13:20,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:13:20,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T21:13:20,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T21:13:20,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:13:20,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:13:20,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T21:13:20,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/nfs.dump.dir in system properties and HBase conf 2024-11-09T21:13:20,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/java.io.tmpdir in system properties and HBase conf 2024-11-09T21:13:20,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:13:20,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T21:13:20,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T21:13:21,851 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T21:13:21,920 INFO [Time-limited test {}] log.Log(170): Logging initialized @2484ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T21:13:21,985 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:13:22,039 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:13:22,059 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:13:22,060 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:13:22,061 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T21:13:22,073 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:13:22,076 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:13:22,077 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:13:22,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/java.io.tmpdir/jetty-localhost-35469-hadoop-hdfs-3_4_1-tests_jar-_-any-11737479396505239256/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:13:22,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:35469} 2024-11-09T21:13:22,273 INFO [Time-limited test {}] server.Server(415): Started @2837ms 2024-11-09T21:13:22,858 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:13:22,868 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:13:22,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:13:22,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:13:22,870 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T21:13:22,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:13:22,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:13:22,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/java.io.tmpdir/jetty-localhost-42889-hadoop-hdfs-3_4_1-tests_jar-_-any-3920164739041534199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:22,977 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:42889} 2024-11-09T21:13:22,977 INFO [Time-limited test {}] server.Server(415): Started @3542ms 2024-11-09T21:13:23,030 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T21:13:23,157 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:13:23,164 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:13:23,167 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:13:23,168 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:13:23,168 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T21:13:23,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:13:23,172 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:13:23,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/java.io.tmpdir/jetty-localhost-42401-hadoop-hdfs-3_4_1-tests_jar-_-any-2985363241653930820/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:23,298 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:42401} 2024-11-09T21:13:23,298 INFO [Time-limited test {}] server.Server(415): Started @3862ms 2024-11-09T21:13:23,301 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T21:13:23,352 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:13:23,358 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:13:23,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:13:23,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:13:23,362 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T21:13:23,363 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:13:23,363 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:13:23,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/java.io.tmpdir/jetty-localhost-34485-hadoop-hdfs-3_4_1-tests_jar-_-any-7045869766569167836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:23,464 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:34485} 2024-11-09T21:13:23,465 INFO [Time-limited test {}] server.Server(415): Started @4029ms 2024-11-09T21:13:23,466 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T21:13:24,937 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data3/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:24,937 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data1/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:24,937 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data4/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:24,937 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data2/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:24,970 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T21:13:24,970 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T21:13:25,020 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7518e4624891adca with lease ID 0x4a4c6f7aaf70c1ea: Processing first storage report for DS-82bf6edd-a5c7-4747-981d-7c0c38502adf from datanode DatanodeRegistration(127.0.0.1:36111, datanodeUuid=411729ce-521a-432a-85e3-327a8e4035f0, infoPort=41797, infoSecurePort=0, ipcPort=33139, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7518e4624891adca with lease ID 0x4a4c6f7aaf70c1ea: from storage DS-82bf6edd-a5c7-4747-981d-7c0c38502adf node DatanodeRegistration(127.0.0.1:36111, datanodeUuid=411729ce-521a-432a-85e3-327a8e4035f0, infoPort=41797, infoSecurePort=0, ipcPort=33139, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26cb2e55d51e7151 with lease ID 0x4a4c6f7aaf70c1e9: Processing first storage report for DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1 from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=aaa29a99-b9e0-4725-9a94-fe5cd18d3bc3, infoPort=46793, infoSecurePort=0, ipcPort=40427, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26cb2e55d51e7151 with lease ID 0x4a4c6f7aaf70c1e9: from storage DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1 node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=aaa29a99-b9e0-4725-9a94-fe5cd18d3bc3, infoPort=46793, infoSecurePort=0, ipcPort=40427, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7518e4624891adca with lease ID 0x4a4c6f7aaf70c1ea: Processing first storage report for DS-6d3503c3-e411-404b-985f-c701f026d4d5 from datanode DatanodeRegistration(127.0.0.1:36111, datanodeUuid=411729ce-521a-432a-85e3-327a8e4035f0, infoPort=41797, infoSecurePort=0, ipcPort=33139, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7518e4624891adca with lease ID 0x4a4c6f7aaf70c1ea: from storage DS-6d3503c3-e411-404b-985f-c701f026d4d5 node DatanodeRegistration(127.0.0.1:36111, datanodeUuid=411729ce-521a-432a-85e3-327a8e4035f0, infoPort=41797, infoSecurePort=0, ipcPort=33139, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26cb2e55d51e7151 with lease ID 0x4a4c6f7aaf70c1e9: Processing first storage report for DS-a9924f01-1931-4810-9e6e-b412ec30a3a4 from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=aaa29a99-b9e0-4725-9a94-fe5cd18d3bc3, infoPort=46793, infoSecurePort=0, ipcPort=40427, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26cb2e55d51e7151 with lease ID 0x4a4c6f7aaf70c1e9: from storage DS-a9924f01-1931-4810-9e6e-b412ec30a3a4 node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=aaa29a99-b9e0-4725-9a94-fe5cd18d3bc3, infoPort=46793, infoSecurePort=0, ipcPort=40427, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,116 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data6/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:25,116 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data5/current/BP-1162922389-172.17.0.2-1731186801332/current, will proceed with Du for space computation calculation, 2024-11-09T21:13:25,139 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T21:13:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd65d1d539bd4f729 with lease ID 0x4a4c6f7aaf70c1eb: Processing first storage report for DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2 from datanode DatanodeRegistration(127.0.0.1:43285, datanodeUuid=190569f7-1ad2-403a-b503-6bf63563d99c, infoPort=33715, infoSecurePort=0, ipcPort=35061, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd65d1d539bd4f729 with lease ID 0x4a4c6f7aaf70c1eb: from storage DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2 node DatanodeRegistration(127.0.0.1:43285, datanodeUuid=190569f7-1ad2-403a-b503-6bf63563d99c, infoPort=33715, infoSecurePort=0, ipcPort=35061, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd65d1d539bd4f729 with lease ID 0x4a4c6f7aaf70c1eb: Processing first storage report for DS-aaf7882d-2166-4d06-a6d7-563dd11a8c87 from datanode DatanodeRegistration(127.0.0.1:43285, datanodeUuid=190569f7-1ad2-403a-b503-6bf63563d99c, infoPort=33715, infoSecurePort=0, ipcPort=35061, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332) 2024-11-09T21:13:25,149 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd65d1d539bd4f729 with lease ID 0x4a4c6f7aaf70c1eb: from storage DS-aaf7882d-2166-4d06-a6d7-563dd11a8c87 node DatanodeRegistration(127.0.0.1:43285, datanodeUuid=190569f7-1ad2-403a-b503-6bf63563d99c, infoPort=33715, infoSecurePort=0, ipcPort=35061, storageInfo=lv=-57;cid=testClusterID;nsid=670466256;c=1731186801332), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:13:25,240 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8 2024-11-09T21:13:25,312 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/zookeeper_0, clientPort=59919, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T21:13:25,322 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59919 2024-11-09T21:13:25,335 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:25,339 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:25,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741825_1001 (size=7) 2024-11-09T21:13:25,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741825_1001 (size=7) 2024-11-09T21:13:25,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741825_1001 (size=7) 2024-11-09T21:13:25,983 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 with version=8 2024-11-09T21:13:25,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/hbase-staging 2024-11-09T21:13:26,096 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T21:13:26,399 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:13:26,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:26,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:26,419 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:13:26,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:26,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:13:26,624 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T21:13:26,701 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T21:13:26,710 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T21:13:26,714 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:13:26,739 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3177 (auto-detected) 2024-11-09T21:13:26,740 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T21:13:26,758 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38717 2024-11-09T21:13:26,779 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38717 connecting to ZooKeeper ensemble=127.0.0.1:59919 2024-11-09T21:13:26,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387170x0, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:13:26,949 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38717-0x101217245da0000 connected 2024-11-09T21:13:27,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,052 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:27,078 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05, hbase.cluster.distributed=false 2024-11-09T21:13:27,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:13:27,119 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-09T21:13:27,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38717 2024-11-09T21:13:27,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38717 2024-11-09T21:13:27,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-09T21:13:27,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-09T21:13:27,252 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:13:27,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,254 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:13:27,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:13:27,257 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T21:13:27,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:13:27,261 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46353 2024-11-09T21:13:27,263 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46353 connecting to ZooKeeper ensemble=127.0.0.1:59919 2024-11-09T21:13:27,263 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,266 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463530x0, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:13:27,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463530x0, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:27,278 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46353-0x101217245da0001 connected 2024-11-09T21:13:27,282 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T21:13:27,292 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T21:13:27,295 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T21:13:27,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:13:27,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46353 2024-11-09T21:13:27,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46353 2024-11-09T21:13:27,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46353 2024-11-09T21:13:27,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46353 2024-11-09T21:13:27,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46353 2024-11-09T21:13:27,317 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:13:27,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,318 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:13:27,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:13:27,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T21:13:27,319 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:13:27,320 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37605 2024-11-09T21:13:27,322 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37605 connecting to ZooKeeper ensemble=127.0.0.1:59919 2024-11-09T21:13:27,322 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,325 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376050x0, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:13:27,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376050x0, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:27,341 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37605-0x101217245da0002 connected 2024-11-09T21:13:27,341 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T21:13:27,343 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T21:13:27,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T21:13:27,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:13:27,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37605 2024-11-09T21:13:27,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37605 2024-11-09T21:13:27,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37605 2024-11-09T21:13:27,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37605 2024-11-09T21:13:27,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37605 2024-11-09T21:13:27,372 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:13:27,372 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T21:13:27,373 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:13:27,374 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39113 2024-11-09T21:13:27,376 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39113 connecting to ZooKeeper ensemble=127.0.0.1:59919 2024-11-09T21:13:27,377 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391130x0, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:13:27,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391130x0, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:27,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39113-0x101217245da0003 connected 2024-11-09T21:13:27,394 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T21:13:27,395 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T21:13:27,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T21:13:27,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:13:27,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39113 2024-11-09T21:13:27,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39113 2024-11-09T21:13:27,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39113 2024-11-09T21:13:27,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39113 2024-11-09T21:13:27,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39113 2024-11-09T21:13:27,436 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9514d6d5e5be:38717 2024-11-09T21:13:27,441 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:27,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,462 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,501 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T21:13:27,504 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9514d6d5e5be,38717,1731186806214 from backup master directory 2024-11-09T21:13:27,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:27,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,531 WARN [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:13:27,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:13:27,531 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:27,534 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T21:13:27,553 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T21:13:27,647 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/hbase.id] with ID: 5a6cc256-0d6e-4cf4-8d84-bd20115e44bf 2024-11-09T21:13:27,647 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/.tmp/hbase.id 2024-11-09T21:13:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741826_1002 (size=42) 2024-11-09T21:13:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741826_1002 (size=42) 2024-11-09T21:13:27,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741826_1002 (size=42) 2024-11-09T21:13:27,686 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/.tmp/hbase.id]:[hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/hbase.id] 2024-11-09T21:13:27,765 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:13:27,773 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T21:13:27,805 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 29ms. 2024-11-09T21:13:27,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:27,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741827_1003 (size=196) 2024-11-09T21:13:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741827_1003 (size=196) 2024-11-09T21:13:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741827_1003 (size=196) 2024-11-09T21:13:27,907 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:13:27,909 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T21:13:27,928 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:27,935 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:27,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741828_1004 (size=1189) 2024-11-09T21:13:27,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741828_1004 (size=1189) 2024-11-09T21:13:27,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741828_1004 (size=1189) 2024-11-09T21:13:28,014 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store 2024-11-09T21:13:28,034 WARN [IPC Server handler 4 on default port 35897 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T21:13:28,034 WARN [IPC Server handler 4 on default port 35897 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T21:13:28,034 WARN [IPC Server handler 4 on default port 35897 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T21:13:28,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741829_1005 (size=34) 2024-11-09T21:13:28,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741829_1005 (size=34) 2024-11-09T21:13:28,061 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T21:13:28,066 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:28,068 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:13:28,068 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:28,068 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:28,071 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:13:28,071 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:28,071 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:28,073 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731186808068Disabling compacts and flushes for region at 1731186808068Disabling writes for close at 1731186808071 (+3 ms)Writing region close event to WAL at 1731186808071Closed at 1731186808071 2024-11-09T21:13:28,076 WARN [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/.initializing 2024-11-09T21:13:28,076 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/WALs/9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:28,092 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:13:28,116 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C38717%2C1731186806214, suffix=, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/WALs/9514d6d5e5be,38717,1731186806214, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/oldWALs, maxLogs=10 2024-11-09T21:13:28,151 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/WALs/9514d6d5e5be,38717,1731186806214/9514d6d5e5be%2C38717%2C1731186806214.1731186808122, exclude list is [], retry=0 2024-11-09T21:13:28,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:28,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T21:13:28,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:28,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:28,242 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/WALs/9514d6d5e5be,38717,1731186806214/9514d6d5e5be%2C38717%2C1731186806214.1731186808122 2024-11-09T21:13:28,245 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46793:46793),(127.0.0.1/127.0.0.1:33715:33715),(127.0.0.1/127.0.0.1:41797:41797)] 2024-11-09T21:13:28,246 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:13:28,246 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:28,251 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,252 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T21:13:28,363 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:28,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:28,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T21:13:28,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:28,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:28,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T21:13:28,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:28,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:28,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T21:13:28,386 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:28,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:28,388 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,394 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,396 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,405 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,406 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,411 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:13:28,417 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:13:28,426 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:28,428 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72816650, jitterRate=0.08505263924598694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:13:28,439 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731186808270Initializing all the Stores at 1731186808273 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186808274 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186808276 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186808277 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186808277Cleaning up temporary data from old regions at 1731186808406 (+129 ms)Region opened successfully at 1731186808439 (+33 ms) 2024-11-09T21:13:28,441 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T21:13:28,484 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b663b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:13:28,524 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T21:13:28,540 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:28,540 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:28,544 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:28,546 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T21:13:28,553 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T21:13:28,554 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T21:13:28,593 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T21:13:28,603 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T21:13:28,666 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T21:13:28,669 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T21:13:28,673 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T21:13:28,685 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T21:13:28,689 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T21:13:28,698 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T21:13:28,709 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T21:13:28,711 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T21:13:28,719 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T21:13:28,767 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T21:13:28,779 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T21:13:28,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:28,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:28,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:28,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:28,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,803 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9514d6d5e5be,38717,1731186806214, sessionid=0x101217245da0000, setting cluster-up flag (Was=false) 2024-11-09T21:13:28,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,866 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T21:13:28,869 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:28,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:28,929 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T21:13:28,931 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:28,937 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T21:13:29,005 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:29,014 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T21:13:29,015 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(746): ClusterId : 5a6cc256-0d6e-4cf4-8d84-bd20115e44bf 2024-11-09T21:13:29,016 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(746): ClusterId : 5a6cc256-0d6e-4cf4-8d84-bd20115e44bf 2024-11-09T21:13:29,018 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(746): ClusterId : 5a6cc256-0d6e-4cf4-8d84-bd20115e44bf 2024-11-09T21:13:29,018 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T21:13:29,018 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T21:13:29,018 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T21:13:29,021 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T21:13:29,027 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9514d6d5e5be,38717,1731186806214 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T21:13:29,045 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T21:13:29,045 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T21:13:29,045 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T21:13:29,045 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T21:13:29,045 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T21:13:29,045 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9514d6d5e5be:0, corePoolSize=10, maxPoolSize=10 2024-11-09T21:13:29,046 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,047 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:13:29,047 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,048 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186839048 2024-11-09T21:13:29,050 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T21:13:29,051 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T21:13:29,052 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:29,053 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T21:13:29,055 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T21:13:29,055 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T21:13:29,056 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T21:13:29,056 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T21:13:29,058 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,057 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,059 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T21:13:29,061 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T21:13:29,063 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T21:13:29,063 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T21:13:29,065 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T21:13:29,066 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T21:13:29,067 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T21:13:29,067 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T21:13:29,068 DEBUG [RS:2;9514d6d5e5be:39113 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183f8075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:13:29,068 DEBUG [RS:1;9514d6d5e5be:37605 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202ccab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:13:29,068 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186809067,5,FailOnTimeoutGroup] 2024-11-09T21:13:29,069 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T21:13:29,069 DEBUG [RS:0;9514d6d5e5be:46353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ee89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:13:29,072 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186809068,5,FailOnTimeoutGroup] 2024-11-09T21:13:29,072 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,073 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T21:13:29,074 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,074 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741831_1007 (size=1321) 2024-11-09T21:13:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741831_1007 (size=1321) 2024-11-09T21:13:29,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741831_1007 (size=1321) 2024-11-09T21:13:29,090 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9514d6d5e5be:39113 2024-11-09T21:13:29,091 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9514d6d5e5be:46353 2024-11-09T21:13:29,091 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9514d6d5e5be:37605 2024-11-09T21:13:29,093 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T21:13:29,093 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:29,094 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T21:13:29,094 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T21:13:29,094 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T21:13:29,095 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T21:13:29,095 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T21:13:29,095 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T21:13:29,095 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T21:13:29,095 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T21:13:29,095 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T21:13:29,098 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(2659): reportForDuty to master=9514d6d5e5be,38717,1731186806214 with port=46353, startcode=1731186807216 2024-11-09T21:13:29,098 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(2659): reportForDuty to master=9514d6d5e5be,38717,1731186806214 with port=39113, startcode=1731186807372 2024-11-09T21:13:29,098 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(2659): reportForDuty to master=9514d6d5e5be,38717,1731186806214 with port=37605, startcode=1731186807317 2024-11-09T21:13:29,111 DEBUG [RS:1;9514d6d5e5be:37605 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T21:13:29,111 DEBUG [RS:0;9514d6d5e5be:46353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T21:13:29,111 DEBUG [RS:2;9514d6d5e5be:39113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T21:13:29,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741832_1008 (size=32) 2024-11-09T21:13:29,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741832_1008 (size=32) 2024-11-09T21:13:29,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741832_1008 (size=32) 2024-11-09T21:13:29,131 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:29,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:13:29,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:13:29,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:13:29,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:13:29,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:13:29,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:13:29,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:13:29,155 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32825, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T21:13:29,155 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51195, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T21:13:29,155 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53749, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T21:13:29,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:13:29,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:13:29,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:29,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:29,163 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,166 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(517): Registering regionserver=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:13:29,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:13:29,167 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:13:29,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:13:29,179 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,180 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(517): Registering regionserver=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,181 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:29,183 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68076216, jitterRate=0.014414668083190918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:13:29,184 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:29,184 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35897 2024-11-09T21:13:29,184 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T21:13:29,187 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,187 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(517): Registering regionserver=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,187 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:29,187 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35897 2024-11-09T21:13:29,188 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T21:13:29,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731186809131Initializing all the Stores at 1731186809133 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809133Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809134 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186809134Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809134Cleaning up temporary data from old regions at 1731186809166 (+32 ms)Region opened successfully at 1731186809190 (+24 ms) 2024-11-09T21:13:29,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:13:29,190 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:13:29,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:13:29,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:13:29,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:13:29,192 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:13:29,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731186809190Disabling compacts and flushes for region at 1731186809190Disabling writes for close at 1731186809191 (+1 ms)Writing region close event to WAL at 1731186809192 (+1 ms)Closed at 1731186809192 2024-11-09T21:13:29,196 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:29,196 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T21:13:29,197 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:29,197 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35897 2024-11-09T21:13:29,197 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T21:13:29,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:13:29,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T21:13:29,217 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:29,221 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T21:13:29,247 DEBUG [RS:2;9514d6d5e5be:39113 {}] zookeeper.ZKUtil(111): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,247 WARN [RS:2;9514d6d5e5be:39113 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:13:29,248 DEBUG [RS:1;9514d6d5e5be:37605 {}] zookeeper.ZKUtil(111): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,248 INFO [RS:2;9514d6d5e5be:39113 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:29,248 WARN [RS:1;9514d6d5e5be:37605 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:13:29,248 INFO [RS:1;9514d6d5e5be:37605 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:29,248 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,248 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,249 DEBUG [RS:0;9514d6d5e5be:46353 {}] zookeeper.ZKUtil(111): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,249 WARN [RS:0;9514d6d5e5be:46353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:13:29,249 INFO [RS:0;9514d6d5e5be:46353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:29,249 DEBUG [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9514d6d5e5be,46353,1731186807216] 2024-11-09T21:13:29,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9514d6d5e5be,39113,1731186807372] 2024-11-09T21:13:29,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9514d6d5e5be,37605,1731186807317] 2024-11-09T21:13:29,277 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T21:13:29,277 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T21:13:29,277 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T21:13:29,295 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T21:13:29,295 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T21:13:29,295 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T21:13:29,302 INFO [RS:2;9514d6d5e5be:39113 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T21:13:29,302 INFO [RS:1;9514d6d5e5be:37605 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T21:13:29,302 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,302 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,303 INFO [RS:0;9514d6d5e5be:46353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T21:13:29,303 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,305 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T21:13:29,306 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T21:13:29,307 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T21:13:29,314 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T21:13:29,314 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T21:13:29,316 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,316 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,316 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T21:13:29,316 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,316 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,316 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,316 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:13:29,317 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,317 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,318 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:2;9514d6d5e5be:39113 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,318 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:0;9514d6d5e5be:46353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,318 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:13:29,319 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,319 DEBUG [RS:1;9514d6d5e5be:37605 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:13:29,324 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,325 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,325 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,325 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,325 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,325 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,39113,1731186807372-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:13:29,328 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,328 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,329 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,329 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,329 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,329 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,37605,1731186807317-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:13:29,332 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,333 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,333 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,333 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,333 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,333 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,46353,1731186807216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:13:29,349 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T21:13:29,351 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,39113,1731186807372-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,352 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,352 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.Replication(171): 9514d6d5e5be,39113,1731186807372 started 2024-11-09T21:13:29,353 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T21:13:29,353 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,37605,1731186807317-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,353 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,353 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.Replication(171): 9514d6d5e5be,37605,1731186807317 started 2024-11-09T21:13:29,354 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T21:13:29,354 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,46353,1731186807216-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,355 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,355 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.Replication(171): 9514d6d5e5be,46353,1731186807216 started 2024-11-09T21:13:29,372 WARN [9514d6d5e5be:38717 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:13:29,375 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,375 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,376 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1482): Serving as 9514d6d5e5be,37605,1731186807317, RpcServer on 9514d6d5e5be/172.17.0.2:37605, sessionid=0x101217245da0002 2024-11-09T21:13:29,376 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1482): Serving as 9514d6d5e5be,39113,1731186807372, RpcServer on 9514d6d5e5be/172.17.0.2:39113, sessionid=0x101217245da0003 2024-11-09T21:13:29,376 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:29,376 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1482): Serving as 9514d6d5e5be,46353,1731186807216, RpcServer on 9514d6d5e5be/172.17.0.2:46353, sessionid=0x101217245da0001 2024-11-09T21:13:29,377 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T21:13:29,377 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T21:13:29,377 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T21:13:29,377 DEBUG [RS:0;9514d6d5e5be:46353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,377 DEBUG [RS:2;9514d6d5e5be:39113 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,377 DEBUG [RS:1;9514d6d5e5be:37605 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,377 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,37605,1731186807317' 2024-11-09T21:13:29,377 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,39113,1731186807372' 2024-11-09T21:13:29,377 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,46353,1731186807216' 2024-11-09T21:13:29,377 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T21:13:29,377 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T21:13:29,377 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T21:13:29,378 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T21:13:29,378 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T21:13:29,378 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T21:13:29,379 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T21:13:29,379 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T21:13:29,379 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T21:13:29,379 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,379 DEBUG [RS:1;9514d6d5e5be:37605 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:29,379 DEBUG [RS:2;9514d6d5e5be:39113 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,46353,1731186807216' 2024-11-09T21:13:29,379 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,37605,1731186807317' 2024-11-09T21:13:29,379 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,39113,1731186807372' 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T21:13:29,379 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T21:13:29,379 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T21:13:29,379 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T21:13:29,380 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T21:13:29,380 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T21:13:29,380 DEBUG [RS:0;9514d6d5e5be:46353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T21:13:29,380 INFO [RS:0;9514d6d5e5be:46353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T21:13:29,380 INFO [RS:0;9514d6d5e5be:46353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T21:13:29,380 DEBUG [RS:1;9514d6d5e5be:37605 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T21:13:29,380 DEBUG [RS:2;9514d6d5e5be:39113 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T21:13:29,380 INFO [RS:1;9514d6d5e5be:37605 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T21:13:29,380 INFO [RS:2;9514d6d5e5be:39113 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T21:13:29,380 INFO [RS:1;9514d6d5e5be:37605 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T21:13:29,380 INFO [RS:2;9514d6d5e5be:39113 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T21:13:29,485 INFO [RS:0;9514d6d5e5be:46353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:13:29,485 INFO [RS:2;9514d6d5e5be:39113 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:13:29,485 INFO [RS:1;9514d6d5e5be:37605 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:13:29,488 INFO [RS:1;9514d6d5e5be:37605 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C37605%2C1731186807317, suffix=, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,37605,1731186807317, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs, maxLogs=32 2024-11-09T21:13:29,488 INFO [RS:2;9514d6d5e5be:39113 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C39113%2C1731186807372, suffix=, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs, maxLogs=32 2024-11-09T21:13:29,488 INFO [RS:0;9514d6d5e5be:46353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C46353%2C1731186807216, suffix=, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs, maxLogs=32 2024-11-09T21:13:29,508 DEBUG [RS:1;9514d6d5e5be:37605 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,37605,1731186807317/9514d6d5e5be%2C37605%2C1731186807317.1731186809493, exclude list is [], retry=0 2024-11-09T21:13:29,513 DEBUG [RS:2;9514d6d5e5be:39113 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372/9514d6d5e5be%2C39113%2C1731186807372.1731186809493, exclude list is [], retry=0 2024-11-09T21:13:29,513 DEBUG [RS:0;9514d6d5e5be:46353 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, exclude list is [], retry=0 2024-11-09T21:13:29,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:29,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:29,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:29,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:29,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:29,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:29,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:29,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:29,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:29,557 INFO [RS:0;9514d6d5e5be:46353 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 2024-11-09T21:13:29,557 INFO [RS:2;9514d6d5e5be:39113 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372/9514d6d5e5be%2C39113%2C1731186807372.1731186809493 2024-11-09T21:13:29,558 DEBUG [RS:0;9514d6d5e5be:46353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46793:46793),(127.0.0.1/127.0.0.1:41797:41797),(127.0.0.1/127.0.0.1:33715:33715)] 2024-11-09T21:13:29,558 DEBUG [RS:2;9514d6d5e5be:39113 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33715:33715),(127.0.0.1/127.0.0.1:46793:46793),(127.0.0.1/127.0.0.1:41797:41797)] 2024-11-09T21:13:29,559 INFO [RS:1;9514d6d5e5be:37605 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,37605,1731186807317/9514d6d5e5be%2C37605%2C1731186807317.1731186809493 2024-11-09T21:13:29,559 DEBUG [RS:1;9514d6d5e5be:37605 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33715:33715),(127.0.0.1/127.0.0.1:46793:46793),(127.0.0.1/127.0.0.1:41797:41797)] 2024-11-09T21:13:29,627 DEBUG [9514d6d5e5be:38717 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-09T21:13:29,637 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(204): Hosts are {9514d6d5e5be=0} racks are {/default-rack=0} 2024-11-09T21:13:29,644 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T21:13:29,644 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T21:13:29,644 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T21:13:29,644 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T21:13:29,644 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T21:13:29,645 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T21:13:29,645 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T21:13:29,645 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T21:13:29,645 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T21:13:29,645 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T21:13:29,651 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:29,657 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,46353,1731186807216, state=OPENING 2024-11-09T21:13:29,718 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T21:13:29,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:29,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:29,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:29,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:29,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:29,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:29,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:29,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:29,731 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:29,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:29,911 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T21:13:29,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46079, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T21:13:29,925 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T21:13:29,925 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:29,925 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T21:13:29,929 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C46353%2C1731186807216.meta, suffix=.meta, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs, maxLogs=32 2024-11-09T21:13:29,945 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, exclude list is [], retry=0 2024-11-09T21:13:29,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:29,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:29,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:29,952 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta 2024-11-09T21:13:29,952 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33715:33715),(127.0.0.1/127.0.0.1:41797:41797),(127.0.0.1/127.0.0.1:46793:46793)] 2024-11-09T21:13:29,953 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:13:29,954 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T21:13:29,956 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T21:13:29,960 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T21:13:29,964 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T21:13:29,964 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:29,965 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T21:13:29,965 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T21:13:29,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:13:29,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:13:29,969 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:13:29,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:13:29,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:13:29,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:13:29,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:13:29,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:13:29,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:29,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:29,977 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:13:29,978 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:29,980 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:29,983 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:13:29,983 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:13:29,984 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:13:29,987 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:13:29,988 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73661680, jitterRate=0.09764456748962402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:13:29,988 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T21:13:29,990 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731186809965Writing region info on filesystem at 1731186809965Initializing all the Stores at 1731186809967 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809967Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809967Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186809967Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186809967Cleaning up temporary data from old regions at 1731186809983 (+16 ms)Running coprocessor post-open hooks at 1731186809988 (+5 ms)Region opened successfully at 1731186809990 (+2 ms) 2024-11-09T21:13:29,995 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731186809900 2024-11-09T21:13:30,005 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T21:13:30,006 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T21:13:30,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:30,010 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,46353,1731186807216, state=OPEN 2024-11-09T21:13:30,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:30,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:30,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:30,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:30,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:30,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:30,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:30,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:30,035 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:30,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T21:13:30,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,46353,1731186807216 in 302 msec 2024-11-09T21:13:30,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T21:13:30,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 838 msec 2024-11-09T21:13:30,049 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:30,050 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T21:13:30,067 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:13:30,068 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1] 2024-11-09T21:13:30,089 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:13:30,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54355, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:13:30,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1500 sec 2024-11-09T21:13:30,116 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731186810116, completionTime=-1 2024-11-09T21:13:30,119 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-09T21:13:30,119 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:30,144 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-09T21:13:30,144 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186870144 2024-11-09T21:13:30,144 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186930144 2024-11-09T21:13:30,144 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 25 msec 2024-11-09T21:13:30,146 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:30,154 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,154 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,154 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,156 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9514d6d5e5be:38717, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,157 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,157 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,162 DEBUG [master/9514d6d5e5be:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T21:13:30,188 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.656sec 2024-11-09T21:13:30,190 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T21:13:30,191 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T21:13:30,193 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T21:13:30,193 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T21:13:30,193 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T21:13:30,194 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:13:30,195 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T21:13:30,199 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T21:13:30,200 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T21:13:30,200 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38717,1731186806214-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:13:30,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28641182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:13:30,227 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T21:13:30,227 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T21:13:30,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,38717,-1 for getting cluster id 2024-11-09T21:13:30,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:13:30,242 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5a6cc256-0d6e-4cf4-8d84-bd20115e44bf' 2024-11-09T21:13:30,244 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:13:30,244 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5a6cc256-0d6e-4cf4-8d84-bd20115e44bf" 2024-11-09T21:13:30,245 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1ff339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:13:30,245 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,38717,-1] 2024-11-09T21:13:30,247 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:13:30,249 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:30,250 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:13:30,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6972ed55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:13:30,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:13:30,260 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1] 2024-11-09T21:13:30,260 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:13:30,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:13:30,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:30,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:13:30,315 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:30,318 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@417d77fb 2024-11-09T21:13:30,319 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:13:30,322 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:13:30,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:13:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:30,352 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:13:30,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-09T21:13:30,355 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:30,358 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:13:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:30,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741837_1013 (size=358) 2024-11-09T21:13:30,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741837_1013 (size=358) 2024-11-09T21:13:30,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741837_1013 (size=358) 2024-11-09T21:13:30,380 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e306ecbfd9d382cf95e6deab029ad4fa, NAME => 'test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,380 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b6d1bef796951894c699e7a9348b65bd, NAME => 'test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,380 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0d3eb0e0d62ca755868b2ca4e5785004, NAME => 'test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741838_1014 (size=45) 2024-11-09T21:13:30,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741838_1014 (size=45) 2024-11-09T21:13:30,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741840_1016 (size=42) 2024-11-09T21:13:30,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741840_1016 (size=42) 2024-11-09T21:13:30,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741840_1016 (size=42) 2024-11-09T21:13:30,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741838_1014 (size=45) 2024-11-09T21:13:30,411 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e306ecbfd9d382cf95e6deab029ad4fa, disabling compactions & flushes 2024-11-09T21:13:30,412 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. after waiting 0 ms 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:30,412 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e306ecbfd9d382cf95e6deab029ad4fa: Waiting for close lock at 1731186810412Disabling compacts and flushes for region at 1731186810412Disabling writes for close at 1731186810412Writing region close event to WAL at 1731186810412Closed at 1731186810412 2024-11-09T21:13:30,412 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b6d1bef796951894c699e7a9348b65bd, disabling compactions & flushes 2024-11-09T21:13:30,412 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:30,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:30,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. after waiting 0 ms 2024-11-09T21:13:30,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:30,413 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 974f175966e3c40bb61e40b6d82efdc2, NAME => 'test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,413 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:30,413 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b6d1bef796951894c699e7a9348b65bd: Waiting for close lock at 1731186810412Disabling compacts and flushes for region at 1731186810412Disabling writes for close at 1731186810413 (+1 ms)Writing region close event to WAL at 1731186810413Closed at 1731186810413 2024-11-09T21:13:30,414 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4b29c44ccc5641890ba911d38607fb16, NAME => 'test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741839_1015 (size=45) 2024-11-09T21:13:30,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741839_1015 (size=45) 2024-11-09T21:13:30,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741839_1015 (size=45) 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0d3eb0e0d62ca755868b2ca4e5785004, disabling compactions & flushes 2024-11-09T21:13:30,425 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. after waiting 0 ms 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:30,425 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:30,425 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0d3eb0e0d62ca755868b2ca4e5785004: Waiting for close lock at 1731186810425Disabling compacts and flushes for region at 1731186810425Disabling writes for close at 1731186810425Writing region close event to WAL at 1731186810425Closed at 1731186810425 2024-11-09T21:13:30,426 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fcbe2f4ebe581c6c9b90065b0416620c, NAME => 'test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741841_1017 (size=45) 2024-11-09T21:13:30,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741841_1017 (size=45) 2024-11-09T21:13:30,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741841_1017 (size=45) 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 974f175966e3c40bb61e40b6d82efdc2, disabling compactions & flushes 2024-11-09T21:13:30,448 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. after waiting 0 ms 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:30,448 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:30,448 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 974f175966e3c40bb61e40b6d82efdc2: Waiting for close lock at 1731186810448Disabling compacts and flushes for region at 1731186810448Disabling writes for close at 1731186810448Writing region close event to WAL at 1731186810448Closed at 1731186810448 2024-11-09T21:13:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741842_1018 (size=45) 2024-11-09T21:13:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741842_1018 (size=45) 2024-11-09T21:13:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741842_1018 (size=45) 2024-11-09T21:13:30,449 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b01a75f865260c525080606af78b0a41, NAME => 'test,ggg,1731186810324.b01a75f865260c525080606af78b0a41.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fcbe2f4ebe581c6c9b90065b0416620c, disabling compactions & flushes 2024-11-09T21:13:30,451 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. after waiting 0 ms 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:30,451 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:30,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fcbe2f4ebe581c6c9b90065b0416620c: Waiting for close lock at 1731186810451Disabling compacts and flushes for region at 1731186810451Disabling writes for close at 1731186810451Writing region close event to WAL at 1731186810451Closed at 1731186810451 2024-11-09T21:13:30,452 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 17cfda7d07986dd9effd1e28ad5cbc35, NAME => 'test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741843_1019 (size=45) 2024-11-09T21:13:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741843_1019 (size=45) 2024-11-09T21:13:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741843_1019 (size=45) 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4b29c44ccc5641890ba911d38607fb16, disabling compactions & flushes 2024-11-09T21:13:30,459 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. after waiting 0 ms 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:30,459 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:30,459 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4b29c44ccc5641890ba911d38607fb16: Waiting for close lock at 1731186810459Disabling compacts and flushes for region at 1731186810459Disabling writes for close at 1731186810459Writing region close event to WAL at 1731186810459Closed at 1731186810459 2024-11-09T21:13:30,460 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 989c28b0ba07f1a68992722b1f5a8b3d, NAME => 'test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741844_1020 (size=45) 2024-11-09T21:13:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741844_1020 (size=45) 2024-11-09T21:13:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741844_1020 (size=45) 2024-11-09T21:13:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:30,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731186810324.b01a75f865260c525080606af78b0a41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,469 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b01a75f865260c525080606af78b0a41, disabling compactions & flushes 2024-11-09T21:13:30,469 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:30,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:30,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. after waiting 0 ms 2024-11-09T21:13:30,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:30,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:30,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b01a75f865260c525080606af78b0a41: Waiting for close lock at 1731186810469Disabling compacts and flushes for region at 1731186810469Disabling writes for close at 1731186810470 (+1 ms)Writing region close event to WAL at 1731186810470Closed at 1731186810470 2024-11-09T21:13:30,471 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e05a8d141de85a4320743bd44a2822ca, NAME => 'test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741845_1021 (size=45) 2024-11-09T21:13:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741845_1021 (size=45) 2024-11-09T21:13:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741845_1021 (size=45) 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 17cfda7d07986dd9effd1e28ad5cbc35, disabling compactions & flushes 2024-11-09T21:13:30,479 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. after waiting 0 ms 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:30,479 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:30,479 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 17cfda7d07986dd9effd1e28ad5cbc35: Waiting for close lock at 1731186810479Disabling compacts and flushes for region at 1731186810479Disabling writes for close at 1731186810479Writing region close event to WAL at 1731186810479Closed at 1731186810479 2024-11-09T21:13:30,480 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ff6038353628d35286993d84ec964b56, NAME => 'test,kkk,1731186810324.ff6038353628d35286993d84ec964b56.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741846_1022 (size=45) 2024-11-09T21:13:30,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741846_1022 (size=45) 2024-11-09T21:13:30,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741846_1022 (size=45) 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 989c28b0ba07f1a68992722b1f5a8b3d, disabling compactions & flushes 2024-11-09T21:13:30,490 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. after waiting 0 ms 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:30,490 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:30,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 989c28b0ba07f1a68992722b1f5a8b3d: Waiting for close lock at 1731186810490Disabling compacts and flushes for region at 1731186810490Disabling writes for close at 1731186810490Writing region close event to WAL at 1731186810490Closed at 1731186810490 2024-11-09T21:13:30,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 121419d26c305ff357f6891790ea3d8f, NAME => 'test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741847_1023 (size=45) 2024-11-09T21:13:30,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741847_1023 (size=45) 2024-11-09T21:13:30,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741847_1023 (size=45) 2024-11-09T21:13:30,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741848_1024 (size=45) 2024-11-09T21:13:30,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741848_1024 (size=45) 2024-11-09T21:13:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741848_1024 (size=45) 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1731186810324.ff6038353628d35286993d84ec964b56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ff6038353628d35286993d84ec964b56, disabling compactions & flushes 2024-11-09T21:13:30,508 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. after waiting 0 ms 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:30,508 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:30,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ff6038353628d35286993d84ec964b56: Waiting for close lock at 1731186810508Disabling compacts and flushes for region at 1731186810508Disabling writes for close at 1731186810508Writing region close event to WAL at 1731186810508Closed at 1731186810508 2024-11-09T21:13:30,509 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 59b46e1f46651edf453a1e428f267809, NAME => 'test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741849_1025 (size=45) 2024-11-09T21:13:30,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741849_1025 (size=45) 2024-11-09T21:13:30,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741849_1025 (size=45) 2024-11-09T21:13:30,514 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,514 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 121419d26c305ff357f6891790ea3d8f, disabling compactions & flushes 2024-11-09T21:13:30,514 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:30,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:30,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. after waiting 0 ms 2024-11-09T21:13:30,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:30,515 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:30,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 121419d26c305ff357f6891790ea3d8f: Waiting for close lock at 1731186810514Disabling compacts and flushes for region at 1731186810514Disabling writes for close at 1731186810515 (+1 ms)Writing region close event to WAL at 1731186810515Closed at 1731186810515 2024-11-09T21:13:30,516 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6999f5a07426b257e5d5577eb9b2495b, NAME => 'test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741850_1026 (size=45) 2024-11-09T21:13:30,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741850_1026 (size=45) 2024-11-09T21:13:30,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741850_1026 (size=45) 2024-11-09T21:13:30,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 59b46e1f46651edf453a1e428f267809, disabling compactions & flushes 2024-11-09T21:13:30,529 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:30,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:30,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. after waiting 0 ms 2024-11-09T21:13:30,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:30,529 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:30,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 59b46e1f46651edf453a1e428f267809: Waiting for close lock at 1731186810529Disabling compacts and flushes for region at 1731186810529Disabling writes for close at 1731186810529Writing region close event to WAL at 1731186810529Closed at 1731186810529 2024-11-09T21:13:30,530 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 00505fb66d17a1a78029487b077f81a5, NAME => 'test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741851_1027 (size=45) 2024-11-09T21:13:30,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741851_1027 (size=45) 2024-11-09T21:13:30,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741851_1027 (size=45) 2024-11-09T21:13:30,538 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,538 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6999f5a07426b257e5d5577eb9b2495b, disabling compactions & flushes 2024-11-09T21:13:30,538 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:30,538 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:30,538 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. after waiting 0 ms 2024-11-09T21:13:30,539 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:30,539 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:30,539 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6999f5a07426b257e5d5577eb9b2495b: Waiting for close lock at 1731186810538Disabling compacts and flushes for region at 1731186810538Disabling writes for close at 1731186810539 (+1 ms)Writing region close event to WAL at 1731186810539Closed at 1731186810539 2024-11-09T21:13:30,539 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => af67ab20ba4ea57284753ee09fac2e5c, NAME => 'test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741852_1028 (size=45) 2024-11-09T21:13:30,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741852_1028 (size=45) 2024-11-09T21:13:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741852_1028 (size=45) 2024-11-09T21:13:30,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 00505fb66d17a1a78029487b077f81a5, disabling compactions & flushes 2024-11-09T21:13:30,548 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:30,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:30,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. after waiting 0 ms 2024-11-09T21:13:30,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:30,548 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:30,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 00505fb66d17a1a78029487b077f81a5: Waiting for close lock at 1731186810547Disabling compacts and flushes for region at 1731186810547Disabling writes for close at 1731186810548 (+1 ms)Writing region close event to WAL at 1731186810548Closed at 1731186810548 2024-11-09T21:13:30,549 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 397ac7d7817cb31ba8717b266a1ae0e9, NAME => 'test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741853_1029 (size=45) 2024-11-09T21:13:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741853_1029 (size=45) 2024-11-09T21:13:30,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741853_1029 (size=45) 2024-11-09T21:13:30,555 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing af67ab20ba4ea57284753ee09fac2e5c, disabling compactions & flushes 2024-11-09T21:13:30,556 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:30,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:30,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. after waiting 0 ms 2024-11-09T21:13:30,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:30,556 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:30,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for af67ab20ba4ea57284753ee09fac2e5c: Waiting for close lock at 1731186810556Disabling compacts and flushes for region at 1731186810556Disabling writes for close at 1731186810556Writing region close event to WAL at 1731186810556Closed at 1731186810556 2024-11-09T21:13:30,557 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7301f6d77020efd526ad58dfb55d8407, NAME => 'test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741854_1030 (size=45) 2024-11-09T21:13:30,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741854_1030 (size=45) 2024-11-09T21:13:30,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741854_1030 (size=45) 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 397ac7d7817cb31ba8717b266a1ae0e9, disabling compactions & flushes 2024-11-09T21:13:30,565 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. after waiting 0 ms 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:30,565 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:30,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 397ac7d7817cb31ba8717b266a1ae0e9: Waiting for close lock at 1731186810565Disabling compacts and flushes for region at 1731186810565Disabling writes for close at 1731186810565Writing region close event to WAL at 1731186810565Closed at 1731186810565 2024-11-09T21:13:30,566 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0e93a9320001bb729736b200e36ec3a0, NAME => 'test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741855_1031 (size=45) 2024-11-09T21:13:30,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741855_1031 (size=45) 2024-11-09T21:13:30,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741855_1031 (size=45) 2024-11-09T21:13:30,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7301f6d77020efd526ad58dfb55d8407, disabling compactions & flushes 2024-11-09T21:13:30,571 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:30,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:30,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. after waiting 0 ms 2024-11-09T21:13:30,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:30,572 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:30,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7301f6d77020efd526ad58dfb55d8407: Waiting for close lock at 1731186810571Disabling compacts and flushes for region at 1731186810571Disabling writes for close at 1731186810572 (+1 ms)Writing region close event to WAL at 1731186810572Closed at 1731186810572 2024-11-09T21:13:30,573 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9030947ba58303ad28b0aa1ef8091a57, NAME => 'test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741856_1032 (size=45) 2024-11-09T21:13:30,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741856_1032 (size=45) 2024-11-09T21:13:30,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741856_1032 (size=45) 2024-11-09T21:13:30,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0e93a9320001bb729736b200e36ec3a0, disabling compactions & flushes 2024-11-09T21:13:30,579 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:30,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:30,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. after waiting 0 ms 2024-11-09T21:13:30,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:30,579 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:30,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0e93a9320001bb729736b200e36ec3a0: Waiting for close lock at 1731186810579Disabling compacts and flushes for region at 1731186810579Disabling writes for close at 1731186810579Writing region close event to WAL at 1731186810579Closed at 1731186810579 2024-11-09T21:13:30,580 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5007b602e53091a64151d33043eac752, NAME => 'test,uuu,1731186810324.5007b602e53091a64151d33043eac752.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741857_1033 (size=45) 2024-11-09T21:13:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741857_1033 (size=45) 2024-11-09T21:13:30,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741857_1033 (size=45) 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9030947ba58303ad28b0aa1ef8091a57, disabling compactions & flushes 2024-11-09T21:13:30,585 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. after waiting 0 ms 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:30,585 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:30,585 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9030947ba58303ad28b0aa1ef8091a57: Waiting for close lock at 1731186810585Disabling compacts and flushes for region at 1731186810585Disabling writes for close at 1731186810585Writing region close event to WAL at 1731186810585Closed at 1731186810585 2024-11-09T21:13:30,586 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 66e1a5bbb9d115a37ba9c2ededd62632, NAME => 'test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741858_1034 (size=45) 2024-11-09T21:13:30,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741858_1034 (size=45) 2024-11-09T21:13:30,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741858_1034 (size=45) 2024-11-09T21:13:30,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731186810324.5007b602e53091a64151d33043eac752.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5007b602e53091a64151d33043eac752, disabling compactions & flushes 2024-11-09T21:13:30,596 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:30,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:30,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731186810324.5007b602e53091a64151d33043eac752. after waiting 0 ms 2024-11-09T21:13:30,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:30,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:30,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5007b602e53091a64151d33043eac752: Waiting for close lock at 1731186810596Disabling compacts and flushes for region at 1731186810596Disabling writes for close at 1731186810596Writing region close event to WAL at 1731186810597 (+1 ms)Closed at 1731186810597 2024-11-09T21:13:30,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 130debef05adf642209cee03bc162eda, NAME => 'test,www,1731186810324.130debef05adf642209cee03bc162eda.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741859_1035 (size=45) 2024-11-09T21:13:30,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741859_1035 (size=45) 2024-11-09T21:13:30,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741859_1035 (size=45) 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 66e1a5bbb9d115a37ba9c2ededd62632, disabling compactions & flushes 2024-11-09T21:13:30,604 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. after waiting 0 ms 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:30,604 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:30,604 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 66e1a5bbb9d115a37ba9c2ededd62632: Waiting for close lock at 1731186810604Disabling compacts and flushes for region at 1731186810604Disabling writes for close at 1731186810604Writing region close event to WAL at 1731186810604Closed at 1731186810604 2024-11-09T21:13:30,605 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 292dc5a8643963551c32ec0807b220c7, NAME => 'test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741860_1036 (size=45) 2024-11-09T21:13:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741860_1036 (size=45) 2024-11-09T21:13:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741860_1036 (size=45) 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731186810324.130debef05adf642209cee03bc162eda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 130debef05adf642209cee03bc162eda, disabling compactions & flushes 2024-11-09T21:13:30,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731186810324.130debef05adf642209cee03bc162eda. after waiting 0 ms 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:30,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:30,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 130debef05adf642209cee03bc162eda: Waiting for close lock at 1731186810609Disabling compacts and flushes for region at 1731186810609Disabling writes for close at 1731186810609Writing region close event to WAL at 1731186810609Closed at 1731186810609 2024-11-09T21:13:30,610 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2efa80434abe3c56dc2511eab9786a84, NAME => 'test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741861_1037 (size=45) 2024-11-09T21:13:30,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741861_1037 (size=45) 2024-11-09T21:13:30,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741861_1037 (size=45) 2024-11-09T21:13:30,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 292dc5a8643963551c32ec0807b220c7, disabling compactions & flushes 2024-11-09T21:13:30,618 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:30,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:30,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. after waiting 0 ms 2024-11-09T21:13:30,618 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:30,618 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:30,619 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 292dc5a8643963551c32ec0807b220c7: Waiting for close lock at 1731186810618Disabling compacts and flushes for region at 1731186810618Disabling writes for close at 1731186810618Writing region close event to WAL at 1731186810618Closed at 1731186810618 2024-11-09T21:13:30,619 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e9fae39eda112797e02faba4a00be94f, NAME => 'test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05 2024-11-09T21:13:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741862_1038 (size=45) 2024-11-09T21:13:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741862_1038 (size=45) 2024-11-09T21:13:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741862_1038 (size=45) 2024-11-09T21:13:30,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2efa80434abe3c56dc2511eab9786a84, disabling compactions & flushes 2024-11-09T21:13:30,624 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:30,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:30,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. after waiting 0 ms 2024-11-09T21:13:30,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:30,625 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:30,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2efa80434abe3c56dc2511eab9786a84: Waiting for close lock at 1731186810624Disabling compacts and flushes for region at 1731186810624Disabling writes for close at 1731186810625 (+1 ms)Writing region close event to WAL at 1731186810625Closed at 1731186810625 2024-11-09T21:13:30,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741863_1039 (size=42) 2024-11-09T21:13:30,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741863_1039 (size=42) 2024-11-09T21:13:30,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741863_1039 (size=42) 2024-11-09T21:13:30,632 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,632 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e9fae39eda112797e02faba4a00be94f, disabling compactions & flushes 2024-11-09T21:13:30,632 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:30,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:30,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. after waiting 0 ms 2024-11-09T21:13:30,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:30,633 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:30,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e9fae39eda112797e02faba4a00be94f: Waiting for close lock at 1731186810632Disabling compacts and flushes for region at 1731186810632Disabling writes for close at 1731186810633 (+1 ms)Writing region close event to WAL at 1731186810633Closed at 1731186810633 2024-11-09T21:13:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:30,900 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:30,900 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e05a8d141de85a4320743bd44a2822ca, disabling compactions & flushes 2024-11-09T21:13:30,900 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:30,900 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:30,900 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. after waiting 0 ms 2024-11-09T21:13:30,901 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:30,901 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:30,901 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e05a8d141de85a4320743bd44a2822ca: Waiting for close lock at 1731186810900Disabling compacts and flushes for region at 1731186810900Disabling writes for close at 1731186810900Writing region close event to WAL at 1731186810901 (+1 ms)Closed at 1731186810901 2024-11-09T21:13:30,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:13:30,913 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,914 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731186810324.b01a75f865260c525080606af78b0a41.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,915 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,915 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,915 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731186810324.ff6038353628d35286993d84ec964b56.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,915 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,915 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731186810324.5007b602e53091a64151d33043eac752.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731186810324.130debef05adf642209cee03bc162eda.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731186810907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186810907"}]},"ts":"1731186810907"} 2024-11-09T21:13:30,958 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-09T21:13:30,960 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:13:30,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186810960"}]},"ts":"1731186810960"} 2024-11-09T21:13:30,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-09T21:13:30,968 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {9514d6d5e5be=0} racks are {/default-rack=0} 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T21:13:30,986 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T21:13:30,986 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T21:13:30,986 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T21:13:30,986 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T21:13:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:30,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e306ecbfd9d382cf95e6deab029ad4fa, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6d1bef796951894c699e7a9348b65bd, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=974f175966e3c40bb61e40b6d82efdc2, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fcbe2f4ebe581c6c9b90065b0416620c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b01a75f865260c525080606af78b0a41, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17cfda7d07986dd9effd1e28ad5cbc35, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e05a8d141de85a4320743bd44a2822ca, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff6038353628d35286993d84ec964b56, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=121419d26c305ff357f6891790ea3d8f, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6999f5a07426b257e5d5577eb9b2495b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=00505fb66d17a1a78029487b077f81a5, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=397ac7d7817cb31ba8717b266a1ae0e9, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7301f6d77020efd526ad58dfb55d8407, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9030947ba58303ad28b0aa1ef8091a57, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5007b602e53091a64151d33043eac752, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=130debef05adf642209cee03bc162eda, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=292dc5a8643963551c32ec0807b220c7, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9fae39eda112797e02faba4a00be94f, ASSIGN}] 2024-11-09T21:13:30,998 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9fae39eda112797e02faba4a00be94f, ASSIGN 2024-11-09T21:13:31,000 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e9fae39eda112797e02faba4a00be94f, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,002 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:31,004 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,005 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=292dc5a8643963551c32ec0807b220c7, ASSIGN 2024-11-09T21:13:31,007 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=292dc5a8643963551c32ec0807b220c7, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,009 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=130debef05adf642209cee03bc162eda, ASSIGN 2024-11-09T21:13:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741829_1005 (size=34) 2024-11-09T21:13:31,011 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=130debef05adf642209cee03bc162eda, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,013 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:31,014 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,016 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5007b602e53091a64151d33043eac752, ASSIGN 2024-11-09T21:13:31,017 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5007b602e53091a64151d33043eac752, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,019 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9030947ba58303ad28b0aa1ef8091a57, ASSIGN 2024-11-09T21:13:31,020 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9030947ba58303ad28b0aa1ef8091a57, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,022 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:31,023 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,024 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7301f6d77020efd526ad58dfb55d8407, ASSIGN 2024-11-09T21:13:31,026 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7301f6d77020efd526ad58dfb55d8407, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,027 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=397ac7d7817cb31ba8717b266a1ae0e9, ASSIGN 2024-11-09T21:13:31,029 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=397ac7d7817cb31ba8717b266a1ae0e9, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,030 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:31,032 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,033 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=00505fb66d17a1a78029487b077f81a5, ASSIGN 2024-11-09T21:13:31,034 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=00505fb66d17a1a78029487b077f81a5, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,036 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6999f5a07426b257e5d5577eb9b2495b, ASSIGN 2024-11-09T21:13:31,037 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6999f5a07426b257e5d5577eb9b2495b, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,039 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:31,040 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,041 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=121419d26c305ff357f6891790ea3d8f, ASSIGN 2024-11-09T21:13:31,043 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=121419d26c305ff357f6891790ea3d8f, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,044 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff6038353628d35286993d84ec964b56, ASSIGN 2024-11-09T21:13:31,045 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff6038353628d35286993d84ec964b56, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e05a8d141de85a4320743bd44a2822ca, ASSIGN 2024-11-09T21:13:31,048 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e05a8d141de85a4320743bd44a2822ca, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:31,050 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,051 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17cfda7d07986dd9effd1e28ad5cbc35, ASSIGN 2024-11-09T21:13:31,052 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17cfda7d07986dd9effd1e28ad5cbc35, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,053 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b01a75f865260c525080606af78b0a41, ASSIGN 2024-11-09T21:13:31,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b01a75f865260c525080606af78b0a41, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,056 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fcbe2f4ebe581c6c9b90065b0416620c, ASSIGN 2024-11-09T21:13:31,057 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fcbe2f4ebe581c6c9b90065b0416620c, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,059 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:31,060 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,061 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=974f175966e3c40bb61e40b6d82efdc2, ASSIGN 2024-11-09T21:13:31,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=974f175966e3c40bb61e40b6d82efdc2, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,064 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6d1bef796951894c699e7a9348b65bd, ASSIGN 2024-11-09T21:13:31,065 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b6d1bef796951894c699e7a9348b65bd, ASSIGN; state=OFFLINE, location=9514d6d5e5be,39113,1731186807372; forceNewPlan=false, retain=false 2024-11-09T21:13:31,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:31,068 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN; state=OFFLINE, location=9514d6d5e5be,46353,1731186807216; forceNewPlan=false, retain=false 2024-11-09T21:13:31,070 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e306ecbfd9d382cf95e6deab029ad4fa, ASSIGN 2024-11-09T21:13:31,072 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e306ecbfd9d382cf95e6deab029ad4fa, ASSIGN; state=OFFLINE, location=9514d6d5e5be,37605,1731186807317; forceNewPlan=false, retain=false 2024-11-09T21:13:31,155 INFO [9514d6d5e5be:38717 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-09T21:13:31,156 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,159 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=59b46e1f46651edf453a1e428f267809, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN because future has completed 2024-11-09T21:13:31,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN because future has completed 2024-11-09T21:13:31,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN because future has completed 2024-11-09T21:13:31,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN because future has completed 2024-11-09T21:13:31,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN because future has completed 2024-11-09T21:13:31,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN because future has completed 2024-11-09T21:13:31,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN because future has completed 2024-11-09T21:13:31,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=292dc5a8643963551c32ec0807b220c7, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5007b602e53091a64151d33043eac752, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7301f6d77020efd526ad58dfb55d8407, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=00505fb66d17a1a78029487b077f81a5, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=121419d26c305ff357f6891790ea3d8f, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN because future has completed 2024-11-09T21:13:31,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e05a8d141de85a4320743bd44a2822ca, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,46353,1731186807216}] 2024-11-09T21:13:31,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=292dc5a8643963551c32ec0807b220c7, ASSIGN because future has completed 2024-11-09T21:13:31,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5007b602e53091a64151d33043eac752, ASSIGN because future has completed 2024-11-09T21:13:31,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7301f6d77020efd526ad58dfb55d8407, ASSIGN because future has completed 2024-11-09T21:13:31,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7301f6d77020efd526ad58dfb55d8407, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=00505fb66d17a1a78029487b077f81a5, ASSIGN because future has completed 2024-11-09T21:13:31,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=121419d26c305ff357f6891790ea3d8f, ASSIGN because future has completed 2024-11-09T21:13:31,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e05a8d141de85a4320743bd44a2822ca, ASSIGN because future has completed 2024-11-09T21:13:31,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17cfda7d07986dd9effd1e28ad5cbc35, ASSIGN because future has completed 2024-11-09T21:13:31,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17cfda7d07986dd9effd1e28ad5cbc35, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e05a8d141de85a4320743bd44a2822ca, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 121419d26c305ff357f6891790ea3d8f, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00505fb66d17a1a78029487b077f81a5, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5007b602e53091a64151d33043eac752, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292dc5a8643963551c32ec0807b220c7, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b6d1bef796951894c699e7a9348b65bd, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e9fae39eda112797e02faba4a00be94f, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=130debef05adf642209cee03bc162eda, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=9030947ba58303ad28b0aa1ef8091a57, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fcbe2f4ebe581c6c9b90065b0416620c, ASSIGN because future has completed 2024-11-09T21:13:31,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcbe2f4ebe581c6c9b90065b0416620c, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b6d1bef796951894c699e7a9348b65bd, ASSIGN because future has completed 2024-11-09T21:13:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e9fae39eda112797e02faba4a00be94f, ASSIGN because future has completed 2024-11-09T21:13:31,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=130debef05adf642209cee03bc162eda, ASSIGN because future has completed 2024-11-09T21:13:31,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 130debef05adf642209cee03bc162eda, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9030947ba58303ad28b0aa1ef8091a57, ASSIGN because future has completed 2024-11-09T21:13:31,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=397ac7d7817cb31ba8717b266a1ae0e9, ASSIGN because future has completed 2024-11-09T21:13:31,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 397ac7d7817cb31ba8717b266a1ae0e9, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9030947ba58303ad28b0aa1ef8091a57, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9fae39eda112797e02faba4a00be94f, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6d1bef796951894c699e7a9348b65bd, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:31,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=6999f5a07426b257e5d5577eb9b2495b, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ff6038353628d35286993d84ec964b56, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b01a75f865260c525080606af78b0a41, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=974f175966e3c40bb61e40b6d82efdc2, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6999f5a07426b257e5d5577eb9b2495b, ASSIGN because future has completed 2024-11-09T21:13:31,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6999f5a07426b257e5d5577eb9b2495b, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff6038353628d35286993d84ec964b56, ASSIGN because future has completed 2024-11-09T21:13:31,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b01a75f865260c525080606af78b0a41, ASSIGN because future has completed 2024-11-09T21:13:31,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=974f175966e3c40bb61e40b6d82efdc2, ASSIGN because future has completed 2024-11-09T21:13:31,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974f175966e3c40bb61e40b6d82efdc2, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e306ecbfd9d382cf95e6deab029ad4fa, ASSIGN because future has completed 2024-11-09T21:13:31,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e306ecbfd9d382cf95e6deab029ad4fa, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b01a75f865260c525080606af78b0a41, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff6038353628d35286993d84ec964b56, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:31,323 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:31,323 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b29c44ccc5641890ba911d38607fb16, NAME => 'test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-09T21:13:31,324 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,324 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,324 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,324 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,326 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,328 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29c44ccc5641890ba911d38607fb16 columnFamilyName family 2024-11-09T21:13:31,328 DEBUG [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,329 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] regionserver.HStore(327): Store=4b29c44ccc5641890ba911d38607fb16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,329 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,331 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,331 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,332 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,332 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,334 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,337 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,338 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4b29c44ccc5641890ba911d38607fb16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70940061, jitterRate=0.05708928406238556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,338 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:31,339 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4b29c44ccc5641890ba911d38607fb16: Running coprocessor pre-open hook at 1731186811324Writing region info on filesystem at 1731186811324Initializing all the Stores at 1731186811325 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811326 (+1 ms)Cleaning up temporary data from old regions at 1731186811332 (+6 ms)Running coprocessor post-open hooks at 1731186811338 (+6 ms)Region opened successfully at 1731186811339 (+1 ms) 2024-11-09T21:13:31,341 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16., pid=33, masterSystemTime=1731186811317 2024-11-09T21:13:31,344 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:31,344 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:31,344 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:31,344 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 59b46e1f46651edf453a1e428f267809, NAME => 'test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-09T21:13:31,345 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,345 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,345 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,345 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4b29c44ccc5641890ba911d38607fb16, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,349 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,351 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59b46e1f46651edf453a1e428f267809 columnFamilyName family 2024-11-09T21:13:31,351 DEBUG [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,352 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] regionserver.HStore(327): Store=59b46e1f46651edf453a1e428f267809/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,352 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,353 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=9 2024-11-09T21:13:31,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,46353,1731186807216 in 172 msec 2024-11-09T21:13:31,354 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,355 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,355 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN in 365 msec 2024-11-09T21:13:31,357 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,360 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,360 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 59b46e1f46651edf453a1e428f267809; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65162391, jitterRate=-0.02900470793247223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,360 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:31,361 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 59b46e1f46651edf453a1e428f267809: Running coprocessor pre-open hook at 1731186811345Writing region info on filesystem at 1731186811345Initializing all the Stores at 1731186811348 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811348Cleaning up temporary data from old regions at 1731186811355 (+7 ms)Running coprocessor post-open hooks at 1731186811361 (+6 ms)Region opened successfully at 1731186811361 2024-11-09T21:13:31,362 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809., pid=35, masterSystemTime=1731186811317 2024-11-09T21:13:31,364 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:31,364 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:31,365 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:31,365 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e1a5bbb9d115a37ba9c2ededd62632, NAME => 'test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-09T21:13:31,365 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,365 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=59b46e1f46651edf453a1e428f267809, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,365 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,365 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,368 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,370 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e1a5bbb9d115a37ba9c2ededd62632 columnFamilyName family 2024-11-09T21:13:31,370 DEBUG [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,371 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T21:13:31,371 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] regionserver.HStore(327): Store=66e1a5bbb9d115a37ba9c2ededd62632/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,371 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,373 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34231, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T21:13:31,373 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,374 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,374 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=17 2024-11-09T21:13:31,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,46353,1731186807216 in 181 msec 2024-11-09T21:13:31,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN in 386 msec 2024-11-09T21:13:31,378 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:31,378 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 00505fb66d17a1a78029487b077f81a5, NAME => 'test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-09T21:13:31,379 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,379 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,379 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,379 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,381 INFO [StoreOpener-00505fb66d17a1a78029487b077f81a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,381 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,384 INFO [StoreOpener-00505fb66d17a1a78029487b077f81a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00505fb66d17a1a78029487b077f81a5 columnFamilyName family 2024-11-09T21:13:31,384 DEBUG [StoreOpener-00505fb66d17a1a78029487b077f81a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,385 INFO [StoreOpener-00505fb66d17a1a78029487b077f81a5-1 {}] regionserver.HStore(327): Store=00505fb66d17a1a78029487b077f81a5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,385 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,386 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,387 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,387 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,387 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,390 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,390 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/66e1a5bbb9d115a37ba9c2ededd62632/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,391 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 66e1a5bbb9d115a37ba9c2ededd62632; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64602279, jitterRate=-0.03735102713108063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,391 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:31,392 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 66e1a5bbb9d115a37ba9c2ededd62632: Running coprocessor pre-open hook at 1731186811365Writing region info on filesystem at 1731186811366 (+1 ms)Initializing all the Stores at 1731186811367 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811367Cleaning up temporary data from old regions at 1731186811374 (+7 ms)Running coprocessor post-open hooks at 1731186811391 (+17 ms)Region opened successfully at 1731186811391 2024-11-09T21:13:31,393 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632., pid=37, masterSystemTime=1731186811317 2024-11-09T21:13:31,394 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/00505fb66d17a1a78029487b077f81a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,396 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 00505fb66d17a1a78029487b077f81a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70383555, jitterRate=0.048796698451042175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,396 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:31,396 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 00505fb66d17a1a78029487b077f81a5: Running coprocessor pre-open hook at 1731186811379Writing region info on filesystem at 1731186811379Initializing all the Stores at 1731186811381 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811381Cleaning up temporary data from old regions at 1731186811387 (+6 ms)Running coprocessor post-open hooks at 1731186811396 (+9 ms)Region opened successfully at 1731186811396 2024-11-09T21:13:31,396 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:31,397 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:31,397 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:31,397 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => af67ab20ba4ea57284753ee09fac2e5c, NAME => 'test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-09T21:13:31,397 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5., pid=43, masterSystemTime=1731186811370 2024-11-09T21:13:31,398 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,398 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,398 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,398 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,398 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,401 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,402 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:31,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,402 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:31,403 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:31,403 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e05a8d141de85a4320743bd44a2822ca, NAME => 'test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-09T21:13:31,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,403 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,404 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,404 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,404 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,406 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af67ab20ba4ea57284753ee09fac2e5c columnFamilyName family 2024-11-09T21:13:31,406 DEBUG [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,407 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] regionserver.HStore(327): Store=af67ab20ba4ea57284753ee09fac2e5c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,407 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00505fb66d17a1a78029487b077f81a5, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,408 INFO [StoreOpener-e05a8d141de85a4320743bd44a2822ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,409 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,410 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,411 INFO [StoreOpener-e05a8d141de85a4320743bd44a2822ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e05a8d141de85a4320743bd44a2822ca columnFamilyName family 2024-11-09T21:13:31,411 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,411 DEBUG [StoreOpener-e05a8d141de85a4320743bd44a2822ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,411 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,412 INFO [StoreOpener-e05a8d141de85a4320743bd44a2822ca-1 {}] regionserver.HStore(327): Store=e05a8d141de85a4320743bd44a2822ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,412 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,413 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,414 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,414 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=26 2024-11-09T21:13:31,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,46353,1731186807216 in 208 msec 2024-11-09T21:13:31,415 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,415 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,417 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/af67ab20ba4ea57284753ee09fac2e5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN in 426 msec 2024-11-09T21:13:31,418 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,418 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened af67ab20ba4ea57284753ee09fac2e5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64163776, jitterRate=-0.043885231018066406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,418 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:31,418 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for af67ab20ba4ea57284753ee09fac2e5c: Running coprocessor pre-open hook at 1731186811398Writing region info on filesystem at 1731186811398Initializing all the Stores at 1731186811401 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811401Cleaning up temporary data from old regions at 1731186811411 (+10 ms)Running coprocessor post-open hooks at 1731186811418 (+7 ms)Region opened successfully at 1731186811418 2024-11-09T21:13:31,420 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c., pid=32, masterSystemTime=1731186811317 2024-11-09T21:13:31,421 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T21:13:31,422 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e05a8d141de85a4320743bd44a2822ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40135, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T21:13:31,423 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e05a8d141de85a4320743bd44a2822ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71314424, jitterRate=0.06266772747039795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,423 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:31,423 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:31,423 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:31,424 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e05a8d141de85a4320743bd44a2822ca: Running coprocessor pre-open hook at 1731186811404Writing region info on filesystem at 1731186811404Initializing all the Stores at 1731186811406 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811406Cleaning up temporary data from old regions at 1731186811415 (+9 ms)Running coprocessor post-open hooks at 1731186811423 (+8 ms)Region opened successfully at 1731186811424 (+1 ms) 2024-11-09T21:13:31,424 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:31,424 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d3eb0e0d62ca755868b2ca4e5785004, NAME => 'test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-09T21:13:31,424 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,424 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,425 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,425 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,425 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca., pid=41, masterSystemTime=1731186811370 2024-11-09T21:13:31,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=19 2024-11-09T21:13:31,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 00505fb66d17a1a78029487b077f81a5, server=9514d6d5e5be,39113,1731186807372 in 177 msec 2024-11-09T21:13:31,428 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,428 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:31,428 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:31,428 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:31,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 121419d26c305ff357f6891790ea3d8f, NAME => 'test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-09T21:13:31,429 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:31,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 397ac7d7817cb31ba8717b266a1ae0e9, NAME => 'test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-09T21:13:31,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,430 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,430 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,430 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,431 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,431 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,431 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3eb0e0d62ca755868b2ca4e5785004 columnFamilyName family 2024-11-09T21:13:31,431 DEBUG [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=00505fb66d17a1a78029487b077f81a5, ASSIGN in 437 msec 2024-11-09T21:13:31,432 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,433 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] regionserver.HStore(327): Store=0d3eb0e0d62ca755868b2ca4e5785004/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,433 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,434 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,434 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,435 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,435 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e05a8d141de85a4320743bd44a2822ca, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,439 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,441 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=test, region=e05a8d141de85a4320743bd44a2822ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T21:13:31,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,444 INFO [StoreOpener-121419d26c305ff357f6891790ea3d8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,444 INFO [StoreOpener-397ac7d7817cb31ba8717b266a1ae0e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,447 INFO [StoreOpener-397ac7d7817cb31ba8717b266a1ae0e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 397ac7d7817cb31ba8717b266a1ae0e9 columnFamilyName family 2024-11-09T21:13:31,447 DEBUG [StoreOpener-397ac7d7817cb31ba8717b266a1ae0e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,447 INFO [StoreOpener-121419d26c305ff357f6891790ea3d8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 121419d26c305ff357f6891790ea3d8f columnFamilyName family 2024-11-09T21:13:31,447 DEBUG [StoreOpener-121419d26c305ff357f6891790ea3d8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,447 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0d3eb0e0d62ca755868b2ca4e5785004/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=14 2024-11-09T21:13:31,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure e05a8d141de85a4320743bd44a2822ca, server=9514d6d5e5be,39113,1731186807372 in 211 msec 2024-11-09T21:13:31,448 INFO [StoreOpener-397ac7d7817cb31ba8717b266a1ae0e9-1 {}] regionserver.HStore(327): Store=397ac7d7817cb31ba8717b266a1ae0e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,449 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,448 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 0d3eb0e0d62ca755868b2ca4e5785004; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64747390, jitterRate=-0.0351887047290802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,449 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:31,449 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 0d3eb0e0d62ca755868b2ca4e5785004: Running coprocessor pre-open hook at 1731186811425Writing region info on filesystem at 1731186811425Initializing all the Stores at 1731186811427 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811427Cleaning up temporary data from old regions at 1731186811435 (+8 ms)Running coprocessor post-open hooks at 1731186811449 (+14 ms)Region opened successfully at 1731186811449 2024-11-09T21:13:31,450 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9514d6d5e5be,46353,1731186807216, table=test, region=af67ab20ba4ea57284753ee09fac2e5c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T21:13:31,450 INFO [StoreOpener-121419d26c305ff357f6891790ea3d8f-1 {}] regionserver.HStore(327): Store=121419d26c305ff357f6891790ea3d8f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,450 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,451 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,451 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004., pid=38, masterSystemTime=1731186811317 2024-11-09T21:13:31,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e05a8d141de85a4320743bd44a2822ca, ASSIGN in 459 msec 2024-11-09T21:13:31,451 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,452 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,454 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,454 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,455 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:31,456 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:31,456 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:31,456 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 2efa80434abe3c56dc2511eab9786a84, NAME => 'test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-09T21:13:31,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,46353,1731186807216 in 282 msec 2024-11-09T21:13:31,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN in 470 msec 2024-11-09T21:13:31,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,466 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/121419d26c305ff357f6891790ea3d8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,467 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 121419d26c305ff357f6891790ea3d8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64680988, jitterRate=-0.03617817163467407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,467 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:31,467 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 121419d26c305ff357f6891790ea3d8f: Running coprocessor pre-open hook at 1731186811430Writing region info on filesystem at 1731186811430Initializing all the Stores at 1731186811435 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811436 (+1 ms)Cleaning up temporary data from old regions at 1731186811454 (+18 ms)Running coprocessor post-open hooks at 1731186811467 (+13 ms)Region opened successfully at 1731186811467 2024-11-09T21:13:31,468 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/397ac7d7817cb31ba8717b266a1ae0e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,469 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f., pid=42, masterSystemTime=1731186811370 2024-11-09T21:13:31,469 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 397ac7d7817cb31ba8717b266a1ae0e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69303720, jitterRate=0.03270590305328369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,469 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:31,469 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 397ac7d7817cb31ba8717b266a1ae0e9: Running coprocessor pre-open hook at 1731186811432Writing region info on filesystem at 1731186811432Initializing all the Stores at 1731186811434 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811434Cleaning up temporary data from old regions at 1731186811453 (+19 ms)Running coprocessor post-open hooks at 1731186811469 (+16 ms)Region opened successfully at 1731186811469 2024-11-09T21:13:31,471 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9., pid=48, masterSystemTime=1731186811421 2024-11-09T21:13:31,472 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:31,472 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:31,472 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:31,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 17cfda7d07986dd9effd1e28ad5cbc35, NAME => 'test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-09T21:13:31,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,475 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:31,475 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:31,475 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:31,475 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 6999f5a07426b257e5d5577eb9b2495b, NAME => 'test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-09T21:13:31,476 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=6 2024-11-09T21:13:31,476 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,46353,1731186807216 in 259 msec 2024-11-09T21:13:31,476 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,476 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,476 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN in 487 msec 2024-11-09T21:13:31,479 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efa80434abe3c56dc2511eab9786a84 columnFamilyName family 2024-11-09T21:13:31,480 DEBUG [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,481 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] regionserver.HStore(327): Store=2efa80434abe3c56dc2511eab9786a84/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,481 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,482 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,484 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 397ac7d7817cb31ba8717b266a1ae0e9, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,484 INFO [StoreOpener-17cfda7d07986dd9effd1e28ad5cbc35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,484 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,485 INFO [StoreOpener-6999f5a07426b257e5d5577eb9b2495b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,488 INFO [StoreOpener-17cfda7d07986dd9effd1e28ad5cbc35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17cfda7d07986dd9effd1e28ad5cbc35 columnFamilyName family 2024-11-09T21:13:31,488 DEBUG [StoreOpener-17cfda7d07986dd9effd1e28ad5cbc35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,488 INFO [StoreOpener-6999f5a07426b257e5d5577eb9b2495b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6999f5a07426b257e5d5577eb9b2495b columnFamilyName family 2024-11-09T21:13:31,488 DEBUG [StoreOpener-6999f5a07426b257e5d5577eb9b2495b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,489 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 121419d26c305ff357f6891790ea3d8f, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,490 INFO [StoreOpener-6999f5a07426b257e5d5577eb9b2495b-1 {}] regionserver.HStore(327): Store=6999f5a07426b257e5d5577eb9b2495b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,490 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,490 INFO [StoreOpener-17cfda7d07986dd9effd1e28ad5cbc35-1 {}] regionserver.HStore(327): Store=17cfda7d07986dd9effd1e28ad5cbc35/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,491 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,491 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,492 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,493 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,493 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,494 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,495 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,496 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-11-09T21:13:31,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 397ac7d7817cb31ba8717b266a1ae0e9, server=9514d6d5e5be,37605,1731186807317 in 219 msec 2024-11-09T21:13:31,497 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,497 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,497 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:31,497 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 2efa80434abe3c56dc2511eab9786a84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63521608, jitterRate=-0.05345427989959717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:31,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 2efa80434abe3c56dc2511eab9786a84: Running coprocessor pre-open hook at 1731186811458Writing region info on filesystem at 1731186811458Initializing all the Stores at 1731186811461 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811461Cleaning up temporary data from old regions at 1731186811484 (+23 ms)Running coprocessor post-open hooks at 1731186811498 (+14 ms)Region opened successfully at 1731186811498 2024-11-09T21:13:31,500 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84., pid=31, masterSystemTime=1731186811317 2024-11-09T21:13:31,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=397ac7d7817cb31ba8717b266a1ae0e9, ASSIGN in 508 msec 2024-11-09T21:13:31,501 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,502 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/6999f5a07426b257e5d5577eb9b2495b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,503 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:31,504 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:31,504 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 6999f5a07426b257e5d5577eb9b2495b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63323571, jitterRate=-0.056405261158943176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,504 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:31,504 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:31,504 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 6999f5a07426b257e5d5577eb9b2495b: Running coprocessor pre-open hook at 1731186811476Writing region info on filesystem at 1731186811476Initializing all the Stores at 1731186811478 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811479 (+1 ms)Cleaning up temporary data from old regions at 1731186811493 (+14 ms)Running coprocessor post-open hooks at 1731186811504 (+11 ms)Region opened successfully at 1731186811504 2024-11-09T21:13:31,504 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 989c28b0ba07f1a68992722b1f5a8b3d, NAME => 'test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-09T21:13:31,504 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,505 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,505 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,505 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,505 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b., pid=52, masterSystemTime=1731186811421 2024-11-09T21:13:31,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=16 2024-11-09T21:13:31,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 121419d26c305ff357f6891790ea3d8f, server=9514d6d5e5be,39113,1731186807372 in 264 msec 2024-11-09T21:13:31,507 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/17cfda7d07986dd9effd1e28ad5cbc35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,507 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 17cfda7d07986dd9effd1e28ad5cbc35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74115283, jitterRate=0.1044037789106369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,508 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:31,508 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 17cfda7d07986dd9effd1e28ad5cbc35: Running coprocessor pre-open hook at 1731186811473Writing region info on filesystem at 1731186811473Initializing all the Stores at 1731186811475 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811476 (+1 ms)Cleaning up temporary data from old regions at 1731186811497 (+21 ms)Running coprocessor post-open hooks at 1731186811508 (+11 ms)Region opened successfully at 1731186811508 2024-11-09T21:13:31,508 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,509 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:31,509 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:31,509 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:31,509 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => e9fae39eda112797e02faba4a00be94f, NAME => 'test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-09T21:13:31,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=121419d26c305ff357f6891790ea3d8f, ASSIGN in 517 msec 2024-11-09T21:13:31,510 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,510 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,510 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,510 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,510 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35., pid=40, masterSystemTime=1731186811370 2024-11-09T21:13:31,512 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 989c28b0ba07f1a68992722b1f5a8b3d columnFamilyName family 2024-11-09T21:13:31,512 DEBUG [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,513 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] regionserver.HStore(327): Store=989c28b0ba07f1a68992722b1f5a8b3d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,513 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,514 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:31,514 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:31,514 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:31,514 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,515 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,515 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => fcbe2f4ebe581c6c9b90065b0416620c, NAME => 'test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-09T21:13:31,516 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6999f5a07426b257e5d5577eb9b2495b, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,516 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,516 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,516 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,517 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,517 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,517 INFO [StoreOpener-e9fae39eda112797e02faba4a00be94f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,519 INFO [StoreOpener-fcbe2f4ebe581c6c9b90065b0416620c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,519 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,523 INFO [StoreOpener-fcbe2f4ebe581c6c9b90065b0416620c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcbe2f4ebe581c6c9b90065b0416620c columnFamilyName family 2024-11-09T21:13:31,523 INFO [StoreOpener-e9fae39eda112797e02faba4a00be94f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9fae39eda112797e02faba4a00be94f columnFamilyName family 2024-11-09T21:13:31,524 DEBUG [StoreOpener-fcbe2f4ebe581c6c9b90065b0416620c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,524 DEBUG [StoreOpener-e9fae39eda112797e02faba4a00be94f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,524 INFO [StoreOpener-fcbe2f4ebe581c6c9b90065b0416620c-1 {}] regionserver.HStore(327): Store=fcbe2f4ebe581c6c9b90065b0416620c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,525 INFO [StoreOpener-e9fae39eda112797e02faba4a00be94f-1 {}] regionserver.HStore(327): Store=e9fae39eda112797e02faba4a00be94f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,525 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,525 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17cfda7d07986dd9effd1e28ad5cbc35, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,526 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,526 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,527 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,527 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,528 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,528 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,528 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,528 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=18 2024-11-09T21:13:31,531 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 6999f5a07426b257e5d5577eb9b2495b, server=9514d6d5e5be,37605,1731186807317 in 235 msec 2024-11-09T21:13:31,532 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,534 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/989c28b0ba07f1a68992722b1f5a8b3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,535 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 989c28b0ba07f1a68992722b1f5a8b3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73158711, jitterRate=0.09014974534511566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,535 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:31,535 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 989c28b0ba07f1a68992722b1f5a8b3d: Running coprocessor pre-open hook at 1731186811505Writing region info on filesystem at 1731186811505Initializing all the Stores at 1731186811508 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811508Cleaning up temporary data from old regions at 1731186811516 (+8 ms)Running coprocessor post-open hooks at 1731186811535 (+19 ms)Region opened successfully at 1731186811535 2024-11-09T21:13:31,537 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e9fae39eda112797e02faba4a00be94f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6999f5a07426b257e5d5577eb9b2495b, ASSIGN in 542 msec 2024-11-09T21:13:31,537 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d., pid=34, masterSystemTime=1731186811317 2024-11-09T21:13:31,538 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened e9fae39eda112797e02faba4a00be94f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64477801, jitterRate=-0.039205893874168396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,538 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:31,538 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/fcbe2f4ebe581c6c9b90065b0416620c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,538 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for e9fae39eda112797e02faba4a00be94f: Running coprocessor pre-open hook at 1731186811510Writing region info on filesystem at 1731186811510Initializing all the Stores at 1731186811513 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811513Cleaning up temporary data from old regions at 1731186811528 (+15 ms)Running coprocessor post-open hooks at 1731186811538 (+10 ms)Region opened successfully at 1731186811538 2024-11-09T21:13:31,539 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened fcbe2f4ebe581c6c9b90065b0416620c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71244488, jitterRate=0.06162559986114502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,539 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:31,539 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for fcbe2f4ebe581c6c9b90065b0416620c: Running coprocessor pre-open hook at 1731186811518Writing region info on filesystem at 1731186811518Initializing all the Stores at 1731186811519 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811519Cleaning up temporary data from old regions at 1731186811528 (+9 ms)Running coprocessor post-open hooks at 1731186811539 (+11 ms)Region opened successfully at 1731186811539 2024-11-09T21:13:31,540 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f., pid=50, masterSystemTime=1731186811421 2024-11-09T21:13:31,540 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c., pid=46, masterSystemTime=1731186811370 2024-11-09T21:13:31,545 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:31,545 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:31,545 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:31,546 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e93a9320001bb729736b200e36ec3a0, NAME => 'test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-09T21:13:31,546 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,546 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,547 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,547 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,547 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:31,547 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:31,547 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:31,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=12 2024-11-09T21:13:31,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 17cfda7d07986dd9effd1e28ad5cbc35, server=9514d6d5e5be,39113,1731186807372 in 315 msec 2024-11-09T21:13:31,547 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => e306ecbfd9d382cf95e6deab029ad4fa, NAME => 'test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-09T21:13:31,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,549 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:31,549 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:31,549 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:31,549 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => b6d1bef796951894c699e7a9348b65bd, NAME => 'test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-09T21:13:31,550 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,550 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,550 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,550 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,553 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,556 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e93a9320001bb729736b200e36ec3a0 columnFamilyName family 2024-11-09T21:13:31,556 DEBUG [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,557 INFO [StoreOpener-b6d1bef796951894c699e7a9348b65bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,557 INFO [StoreOpener-e306ecbfd9d382cf95e6deab029ad4fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17cfda7d07986dd9effd1e28ad5cbc35, ASSIGN in 559 msec 2024-11-09T21:13:31,558 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] regionserver.HStore(327): Store=0e93a9320001bb729736b200e36ec3a0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,558 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,559 INFO [StoreOpener-e306ecbfd9d382cf95e6deab029ad4fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e306ecbfd9d382cf95e6deab029ad4fa columnFamilyName family 2024-11-09T21:13:31,559 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,559 DEBUG [StoreOpener-e306ecbfd9d382cf95e6deab029ad4fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,560 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,560 INFO [StoreOpener-e306ecbfd9d382cf95e6deab029ad4fa-1 {}] regionserver.HStore(327): Store=e306ecbfd9d382cf95e6deab029ad4fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,560 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,560 INFO [StoreOpener-b6d1bef796951894c699e7a9348b65bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6d1bef796951894c699e7a9348b65bd columnFamilyName family 2024-11-09T21:13:31,561 DEBUG [StoreOpener-b6d1bef796951894c699e7a9348b65bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,561 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,561 INFO [StoreOpener-b6d1bef796951894c699e7a9348b65bd-1 {}] regionserver.HStore(327): Store=b6d1bef796951894c699e7a9348b65bd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,561 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,562 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,562 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,562 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,562 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,563 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,563 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,563 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,563 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcbe2f4ebe581c6c9b90065b0416620c, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,564 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,564 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,565 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,566 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,567 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,570 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e306ecbfd9d382cf95e6deab029ad4fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,571 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened e306ecbfd9d382cf95e6deab029ad4fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65206571, jitterRate=-0.02834637463092804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,571 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:31,571 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for e306ecbfd9d382cf95e6deab029ad4fa: Running coprocessor pre-open hook at 1731186811549Writing region info on filesystem at 1731186811549Initializing all the Stores at 1731186811553 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811553Cleaning up temporary data from old regions at 1731186811563 (+10 ms)Running coprocessor post-open hooks at 1731186811571 (+8 ms)Region opened successfully at 1731186811571 2024-11-09T21:13:31,571 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,572 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0e93a9320001bb729736b200e36ec3a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63100079, jitterRate=-0.05973555147647858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,572 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:31,572 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa., pid=54, masterSystemTime=1731186811421 2024-11-09T21:13:31,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-11-09T21:13:31,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fcbe2f4ebe581c6c9b90065b0416620c, server=9514d6d5e5be,39113,1731186807372 in 306 msec 2024-11-09T21:13:31,572 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0e93a9320001bb729736b200e36ec3a0: Running coprocessor pre-open hook at 1731186811547Writing region info on filesystem at 1731186811547Initializing all the Stores at 1731186811551 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811551Cleaning up temporary data from old regions at 1731186811562 (+11 ms)Running coprocessor post-open hooks at 1731186811572 (+10 ms)Region opened successfully at 1731186811572 2024-11-09T21:13:31,573 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b6d1bef796951894c699e7a9348b65bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,574 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0., pid=36, masterSystemTime=1731186811317 2024-11-09T21:13:31,575 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened b6d1bef796951894c699e7a9348b65bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73372512, jitterRate=0.09333562850952148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,575 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:31,575 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for b6d1bef796951894c699e7a9348b65bd: Running coprocessor pre-open hook at 1731186811550Writing region info on filesystem at 1731186811550Initializing all the Stores at 1731186811554 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811554Cleaning up temporary data from old regions at 1731186811564 (+10 ms)Running coprocessor post-open hooks at 1731186811575 (+11 ms)Region opened successfully at 1731186811575 2024-11-09T21:13:31,576 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:31,576 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:31,577 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:31,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fcbe2f4ebe581c6c9b90065b0416620c, ASSIGN in 584 msec 2024-11-09T21:13:31,577 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => b01a75f865260c525080606af78b0a41, NAME => 'test,ggg,1731186810324.b01a75f865260c525080606af78b0a41.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-09T21:13:31,577 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:31,577 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:31,577 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,577 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ggg,1731186810324.b01a75f865260c525080606af78b0a41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,578 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,578 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,578 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd., pid=51, masterSystemTime=1731186811370 2024-11-09T21:13:31,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0e93a9320001bb729736b200e36ec3a0, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,580 INFO [StoreOpener-b01a75f865260c525080606af78b0a41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,582 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:31,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,582 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:31,582 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:31,582 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 7301f6d77020efd526ad58dfb55d8407, NAME => 'test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-09T21:13:31,583 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,583 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,583 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,583 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,584 INFO [StoreOpener-b01a75f865260c525080606af78b0a41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b01a75f865260c525080606af78b0a41 columnFamilyName family 2024-11-09T21:13:31,585 DEBUG [StoreOpener-b01a75f865260c525080606af78b0a41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,585 INFO [StoreOpener-7301f6d77020efd526ad58dfb55d8407-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,586 INFO [StoreOpener-b01a75f865260c525080606af78b0a41-1 {}] regionserver.HStore(327): Store=b01a75f865260c525080606af78b0a41/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,586 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,587 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6d1bef796951894c699e7a9348b65bd, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,589 INFO [StoreOpener-7301f6d77020efd526ad58dfb55d8407-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7301f6d77020efd526ad58dfb55d8407 columnFamilyName family 2024-11-09T21:13:31,589 DEBUG [StoreOpener-7301f6d77020efd526ad58dfb55d8407-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,589 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,589 INFO [StoreOpener-7301f6d77020efd526ad58dfb55d8407-1 {}] regionserver.HStore(327): Store=7301f6d77020efd526ad58dfb55d8407/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,590 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,590 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,590 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,591 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,592 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,592 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,592 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,593 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=23 2024-11-09T21:13:31,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,46353,1731186807216 in 395 msec 2024-11-09T21:13:31,596 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,597 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b01a75f865260c525080606af78b0a41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,599 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened b01a75f865260c525080606af78b0a41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61865392, jitterRate=-0.07813382148742676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,599 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:31,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN in 606 msec 2024-11-09T21:13:31,599 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for b01a75f865260c525080606af78b0a41: Running coprocessor pre-open hook at 1731186811578Writing region info on filesystem at 1731186811578Initializing all the Stores at 1731186811580 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811580Cleaning up temporary data from old regions at 1731186811590 (+10 ms)Running coprocessor post-open hooks at 1731186811599 (+9 ms)Region opened successfully at 1731186811599 2024-11-09T21:13:31,599 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/7301f6d77020efd526ad58dfb55d8407/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,600 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 7301f6d77020efd526ad58dfb55d8407; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65133292, jitterRate=-0.029438316822052002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,600 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:31,600 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731186810324.b01a75f865260c525080606af78b0a41., pid=55, masterSystemTime=1731186811421 2024-11-09T21:13:31,601 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 7301f6d77020efd526ad58dfb55d8407: Running coprocessor pre-open hook at 1731186811583Writing region info on filesystem at 1731186811583Initializing all the Stores at 1731186811584 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811585 (+1 ms)Cleaning up temporary data from old regions at 1731186811592 (+7 ms)Running coprocessor post-open hooks at 1731186811601 (+9 ms)Region opened successfully at 1731186811601 2024-11-09T21:13:31,603 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407., pid=39, masterSystemTime=1731186811370 2024-11-09T21:13:31,604 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:31,604 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:31,604 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:31,605 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 9030947ba58303ad28b0aa1ef8091a57, NAME => 'test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-09T21:13:31,605 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,605 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,605 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,605 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,607 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:31,607 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:31,607 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:31,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=7 2024-11-09T21:13:31,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b6d1bef796951894c699e7a9348b65bd, server=9514d6d5e5be,39113,1731186807372 in 317 msec 2024-11-09T21:13:31,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 5007b602e53091a64151d33043eac752, NAME => 'test,uuu,1731186810324.5007b602e53091a64151d33043eac752.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-09T21:13:31,608 INFO [StoreOpener-9030947ba58303ad28b0aa1ef8091a57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,uuu,1731186810324.5007b602e53091a64151d33043eac752.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b6d1bef796951894c699e7a9348b65bd, ASSIGN in 619 msec 2024-11-09T21:13:31,611 INFO [StoreOpener-5007b602e53091a64151d33043eac752-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,611 INFO [StoreOpener-9030947ba58303ad28b0aa1ef8091a57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9030947ba58303ad28b0aa1ef8091a57 columnFamilyName family 2024-11-09T21:13:31,611 DEBUG [StoreOpener-9030947ba58303ad28b0aa1ef8091a57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,612 INFO [StoreOpener-9030947ba58303ad28b0aa1ef8091a57-1 {}] regionserver.HStore(327): Store=9030947ba58303ad28b0aa1ef8091a57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,613 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,614 INFO [StoreOpener-5007b602e53091a64151d33043eac752-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5007b602e53091a64151d33043eac752 columnFamilyName family 2024-11-09T21:13:31,614 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,615 DEBUG [StoreOpener-5007b602e53091a64151d33043eac752-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,615 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,615 INFO [StoreOpener-5007b602e53091a64151d33043eac752-1 {}] regionserver.HStore(327): Store=5007b602e53091a64151d33043eac752/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7301f6d77020efd526ad58dfb55d8407, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,616 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,616 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,616 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b01a75f865260c525080606af78b0a41, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,618 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,618 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b01a75f865260c525080606af78b0a41, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,624 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,624 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/9030947ba58303ad28b0aa1ef8091a57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,625 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 9030947ba58303ad28b0aa1ef8091a57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70819509, jitterRate=0.055292919278144836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,625 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:31,625 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 9030947ba58303ad28b0aa1ef8091a57: Running coprocessor pre-open hook at 1731186811606Writing region info on filesystem at 1731186811606Initializing all the Stores at 1731186811607 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811607Cleaning up temporary data from old regions at 1731186811616 (+9 ms)Running coprocessor post-open hooks at 1731186811625 (+9 ms)Region opened successfully at 1731186811625 2024-11-09T21:13:31,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=22 2024-11-09T21:13:31,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 7301f6d77020efd526ad58dfb55d8407, server=9514d6d5e5be,39113,1731186807372 in 405 msec 2024-11-09T21:13:31,626 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57., pid=49, masterSystemTime=1731186811421 2024-11-09T21:13:31,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7301f6d77020efd526ad58dfb55d8407, ASSIGN in 636 msec 2024-11-09T21:13:31,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:31,629 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:31,629 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:31,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => ff6038353628d35286993d84ec964b56, NAME => 'test,kkk,1731186810324.ff6038353628d35286993d84ec964b56.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-09T21:13:31,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,kkk,1731186810324.ff6038353628d35286993d84ec964b56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,630 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,631 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/5007b602e53091a64151d33043eac752/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,632 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 5007b602e53091a64151d33043eac752; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62884217, jitterRate=-0.06295214593410492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,633 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:31,633 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 5007b602e53091a64151d33043eac752: Running coprocessor pre-open hook at 1731186811609Writing region info on filesystem at 1731186811609Initializing all the Stores at 1731186811610 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811610Cleaning up temporary data from old regions at 1731186811619 (+9 ms)Running coprocessor post-open hooks at 1731186811633 (+14 ms)Region opened successfully at 1731186811633 2024-11-09T21:13:31,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9030947ba58303ad28b0aa1ef8091a57, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,634 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731186810324.5007b602e53091a64151d33043eac752., pid=44, masterSystemTime=1731186811370 2024-11-09T21:13:31,637 INFO [StoreOpener-ff6038353628d35286993d84ec964b56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,639 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:31,639 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:31,639 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:31,639 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 292dc5a8643963551c32ec0807b220c7, NAME => 'test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-09T21:13:31,639 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,640 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,640 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,640 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,641 INFO [StoreOpener-ff6038353628d35286993d84ec964b56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff6038353628d35286993d84ec964b56 columnFamilyName family 2024-11-09T21:13:31,641 DEBUG [StoreOpener-ff6038353628d35286993d84ec964b56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=11 2024-11-09T21:13:31,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b01a75f865260c525080606af78b0a41, server=9514d6d5e5be,37605,1731186807317 in 330 msec 2024-11-09T21:13:31,642 INFO [StoreOpener-ff6038353628d35286993d84ec964b56-1 {}] regionserver.HStore(327): Store=ff6038353628d35286993d84ec964b56/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,642 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,643 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,643 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b01a75f865260c525080606af78b0a41, ASSIGN in 652 msec 2024-11-09T21:13:31,644 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,644 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,644 INFO [StoreOpener-292dc5a8643963551c32ec0807b220c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5007b602e53091a64151d33043eac752, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,646 INFO [StoreOpener-292dc5a8643963551c32ec0807b220c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292dc5a8643963551c32ec0807b220c7 columnFamilyName family 2024-11-09T21:13:31,647 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,647 DEBUG [StoreOpener-292dc5a8643963551c32ec0807b220c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,647 INFO [StoreOpener-292dc5a8643963551c32ec0807b220c7-1 {}] regionserver.HStore(327): Store=292dc5a8643963551c32ec0807b220c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,647 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,648 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,649 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,650 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,650 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5007b602e53091a64151d33043eac752, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,652 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,652 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/ff6038353628d35286993d84ec964b56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,653 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened ff6038353628d35286993d84ec964b56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75105830, jitterRate=0.11916407942771912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,653 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:31,653 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for ff6038353628d35286993d84ec964b56: Running coprocessor pre-open hook at 1731186811631Writing region info on filesystem at 1731186811631Initializing all the Stores at 1731186811633 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811633Cleaning up temporary data from old regions at 1731186811644 (+11 ms)Running coprocessor post-open hooks at 1731186811653 (+9 ms)Region opened successfully at 1731186811653 2024-11-09T21:13:31,655 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731186810324.ff6038353628d35286993d84ec964b56., pid=56, masterSystemTime=1731186811421 2024-11-09T21:13:31,657 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9514d6d5e5be,37605,1731186807317, table=test, region=9030947ba58303ad28b0aa1ef8091a57. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T21:13:31,658 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/292dc5a8643963551c32ec0807b220c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,660 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 292dc5a8643963551c32ec0807b220c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68263538, jitterRate=0.01720598340034485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,660 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:31,660 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 292dc5a8643963551c32ec0807b220c7: Running coprocessor pre-open hook at 1731186811640Writing region info on filesystem at 1731186811640Initializing all the Stores at 1731186811642 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811642Cleaning up temporary data from old regions at 1731186811650 (+8 ms)Running coprocessor post-open hooks at 1731186811660 (+10 ms)Region opened successfully at 1731186811660 2024-11-09T21:13:31,661 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:31,661 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:31,661 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:31,661 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 974f175966e3c40bb61e40b6d82efdc2, NAME => 'test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-09T21:13:31,661 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,661 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,662 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,662 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=25 2024-11-09T21:13:31,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5007b602e53091a64151d33043eac752, server=9514d6d5e5be,39113,1731186807372 in 406 msec 2024-11-09T21:13:31,665 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7., pid=45, masterSystemTime=1731186811370 2024-11-09T21:13:31,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5007b602e53091a64151d33043eac752, ASSIGN in 674 msec 2024-11-09T21:13:31,667 INFO [StoreOpener-974f175966e3c40bb61e40b6d82efdc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ff6038353628d35286993d84ec964b56, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,668 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:31,668 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:31,670 INFO [StoreOpener-974f175966e3c40bb61e40b6d82efdc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974f175966e3c40bb61e40b6d82efdc2 columnFamilyName family 2024-11-09T21:13:31,670 DEBUG [StoreOpener-974f175966e3c40bb61e40b6d82efdc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,671 INFO [StoreOpener-974f175966e3c40bb61e40b6d82efdc2-1 {}] regionserver.HStore(327): Store=974f175966e3c40bb61e40b6d82efdc2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff6038353628d35286993d84ec964b56, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,671 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:31,673 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,675 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,675 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,675 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292dc5a8643963551c32ec0807b220c7, server=9514d6d5e5be,39113,1731186807372 because future has completed 2024-11-09T21:13:31,678 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=15 2024-11-09T21:13:31,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure ff6038353628d35286993d84ec964b56, server=9514d6d5e5be,37605,1731186807317 in 365 msec 2024-11-09T21:13:31,682 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/974f175966e3c40bb61e40b6d82efdc2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,683 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 974f175966e3c40bb61e40b6d82efdc2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75392967, jitterRate=0.12344275414943695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,683 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:31,683 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 974f175966e3c40bb61e40b6d82efdc2: Running coprocessor pre-open hook at 1731186811662Writing region info on filesystem at 1731186811662Initializing all the Stores at 1731186811663 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811663Cleaning up temporary data from old regions at 1731186811675 (+12 ms)Running coprocessor post-open hooks at 1731186811683 (+8 ms)Region opened successfully at 1731186811683 2024-11-09T21:13:31,684 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2., pid=53, masterSystemTime=1731186811421 2024-11-09T21:13:31,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ff6038353628d35286993d84ec964b56, ASSIGN in 692 msec 2024-11-09T21:13:31,688 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:31,688 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:31,688 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:31,689 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 130debef05adf642209cee03bc162eda, NAME => 'test,www,1731186810324.130debef05adf642209cee03bc162eda.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-09T21:13:31,689 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,689 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,www,1731186810324.130debef05adf642209cee03bc162eda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:31,689 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,690 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=28 2024-11-09T21:13:31,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 292dc5a8643963551c32ec0807b220c7, server=9514d6d5e5be,39113,1731186807372 in 434 msec 2024-11-09T21:13:31,692 INFO [StoreOpener-130debef05adf642209cee03bc162eda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=292dc5a8643963551c32ec0807b220c7, ASSIGN in 701 msec 2024-11-09T21:13:31,694 INFO [StoreOpener-130debef05adf642209cee03bc162eda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130debef05adf642209cee03bc162eda columnFamilyName family 2024-11-09T21:13:31,694 DEBUG [StoreOpener-130debef05adf642209cee03bc162eda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:31,695 INFO [StoreOpener-130debef05adf642209cee03bc162eda-1 {}] regionserver.HStore(327): Store=130debef05adf642209cee03bc162eda/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:31,696 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,697 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,697 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,698 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,698 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974f175966e3c40bb61e40b6d82efdc2, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,701 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=24 2024-11-09T21:13:31,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 9030947ba58303ad28b0aa1ef8091a57, server=9514d6d5e5be,37605,1731186807317 in 424 msec 2024-11-09T21:13:31,705 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/130debef05adf642209cee03bc162eda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:13:31,706 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 130debef05adf642209cee03bc162eda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66466132, jitterRate=-0.009577453136444092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:31,706 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:31,707 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 130debef05adf642209cee03bc162eda: Running coprocessor pre-open hook at 1731186811690Writing region info on filesystem at 1731186811690Initializing all the Stores at 1731186811691 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186811691Cleaning up temporary data from old regions at 1731186811698 (+7 ms)Running coprocessor post-open hooks at 1731186811706 (+8 ms)Region opened successfully at 1731186811706 2024-11-09T21:13:31,708 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731186810324.130debef05adf642209cee03bc162eda., pid=47, masterSystemTime=1731186811421 2024-11-09T21:13:31,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9030947ba58303ad28b0aa1ef8091a57, ASSIGN in 715 msec 2024-11-09T21:13:31,711 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:31,711 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:31,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=8 2024-11-09T21:13:31,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 974f175966e3c40bb61e40b6d82efdc2, server=9514d6d5e5be,37605,1731186807317 in 414 msec 2024-11-09T21:13:31,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=974f175966e3c40bb61e40b6d82efdc2, ASSIGN in 725 msec 2024-11-09T21:13:31,718 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=130debef05adf642209cee03bc162eda, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 130debef05adf642209cee03bc162eda, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e306ecbfd9d382cf95e6deab029ad4fa, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=27 2024-11-09T21:13:31,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 130debef05adf642209cee03bc162eda, server=9514d6d5e5be,37605,1731186807317 in 457 msec 2024-11-09T21:13:31,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=130debef05adf642209cee03bc162eda, ASSIGN in 738 msec 2024-11-09T21:13:31,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-09T21:13:31,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e306ecbfd9d382cf95e6deab029ad4fa, server=9514d6d5e5be,37605,1731186807317 in 430 msec 2024-11-09T21:13:31,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e306ecbfd9d382cf95e6deab029ad4fa, ASSIGN in 743 msec 2024-11-09T21:13:31,735 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:31,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9fae39eda112797e02faba4a00be94f, server=9514d6d5e5be,37605,1731186807317 because future has completed 2024-11-09T21:13:31,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=30 2024-11-09T21:13:31,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e9fae39eda112797e02faba4a00be94f, server=9514d6d5e5be,37605,1731186807317 in 463 msec 2024-11-09T21:13:31,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e9fae39eda112797e02faba4a00be94f, ASSIGN in 756 msec 2024-11-09T21:13:31,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=13 2024-11-09T21:13:31,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,46353,1731186807216 in 564 msec 2024-11-09T21:13:31,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN in 761 msec 2024-11-09T21:13:31,753 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2efa80434abe3c56dc2511eab9786a84, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:31,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,46353,1731186807216 because future has completed 2024-11-09T21:13:31,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-09T21:13:31,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,46353,1731186807216 in 594 msec 2024-11-09T21:13:31,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=4 2024-11-09T21:13:31,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN in 783 msec 2024-11-09T21:13:31,777 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:13:31,777 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186811777"}]},"ts":"1731186811777"} 2024-11-09T21:13:31,779 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-09T21:13:31,781 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:13:31,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4420 sec 2024-11-09T21:13:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:13:32,519 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-09T21:13:32,519 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-09T21:13:32,521 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-09T21:13:32,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-09T21:13:32,548 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,554 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T21:13:32,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T21:13:32,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:13:32,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:13:32,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:13:32,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:13:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e306ecbfd9d382cf95e6deab029ad4fa changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 974f175966e3c40bb61e40b6d82efdc2 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b01a75f865260c525080606af78b0a41 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff6038353628d35286993d84ec964b56 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6999f5a07426b257e5d5577eb9b2495b changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 397ac7d7817cb31ba8717b266a1ae0e9 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9030947ba58303ad28b0aa1ef8091a57 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 130debef05adf642209cee03bc162eda changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e9fae39eda112797e02faba4a00be94f changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b6d1bef796951894c699e7a9348b65bd changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fcbe2f4ebe581c6c9b90065b0416620c changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17cfda7d07986dd9effd1e28ad5cbc35 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e05a8d141de85a4320743bd44a2822ca changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 121419d26c305ff357f6891790ea3d8f changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 00505fb66d17a1a78029487b077f81a5 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7301f6d77020efd526ad58dfb55d8407 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5007b602e53091a64151d33043eac752 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 292dc5a8643963551c32ec0807b220c7 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0d3eb0e0d62ca755868b2ca4e5785004 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b29c44ccc5641890ba911d38607fb16 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 989c28b0ba07f1a68992722b1f5a8b3d changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 59b46e1f46651edf453a1e428f267809 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region af67ab20ba4ea57284753ee09fac2e5c changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e93a9320001bb729736b200e36ec3a0 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 66e1a5bbb9d115a37ba9c2ededd62632 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2efa80434abe3c56dc2511eab9786a84 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:13:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-09T21:13:32,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-09T21:13:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-09T21:13:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(204): Hosts are {9514d6d5e5be=0} racks are {/default-rack=0} 2024-11-09T21:13:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-09T21:13:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-09T21:13:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-09T21:13:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T21:13:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T21:13:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T21:13:32,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T21:13:32,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T21:13:32,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T21:13:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-09T21:13:32,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-09T21:13:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-09T21:13:32,677 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-09T21:13:32,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=384, OpenFileDescriptor=639, MaxFileDescriptor=1048576, SystemLoadAverage=535, ProcessCount=11, AvailableMemoryMB=3659 2024-11-09T21:13:32,706 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-09T21:13:32,707 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-09T21:13:32,707 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:32,708 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9514d6d5e5be,46353,1731186807216: Simulated kill ***** 2024-11-09T21:13:32,708 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-09T21:13:32,710 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-09T21:13:32,712 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-09T21:13:32,713 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-09T21:13:32,713 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-09T21:13:32,713 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 303354112 }, "NonHeapMemoryUsage": { "committed": 151912448, "init": 7667712, "max": -1, "used": 149671936 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be" } ] } 2024-11-09T21:13:32,716 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] master.MasterRpcServices(700): 9514d6d5e5be,46353,1731186807216 reported a fatal error: ***** ABORTING region server 9514d6d5e5be,46353,1731186807216: Simulated kill ***** 2024-11-09T21:13:32,719 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9514d6d5e5be,46353,1731186807216' ***** 2024-11-09T21:13:32,719 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-09T21:13:32,720 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T21:13:32,720 INFO [RS:0;9514d6d5e5be:46353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-09T21:13:32,720 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T21:13:32,720 INFO [RS:0;9514d6d5e5be:46353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-09T21:13:32,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,720 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:13:32,720 INFO [RS:0;9514d6d5e5be:46353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9514d6d5e5be:46353. 2024-11-09T21:13:32,721 DEBUG [RS:0;9514d6d5e5be:46353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:13:32,727 DEBUG [RS:0;9514d6d5e5be:46353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:32,736 INFO [regionserver/9514d6d5e5be:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741836_1012 (size=32114) 2024-11-09T21:13:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741836_1012 (size=32114) 2024-11-09T21:13:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741836_1012 (size=32114) 2024-11-09T21:13:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741835_1011 (size=2237) 2024-11-09T21:13:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741835_1011 (size=2237) 2024-11-09T21:13:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741835_1011 (size=2237) 2024-11-09T21:13:32,766 DEBUG [RS:0;9514d6d5e5be:46353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:32,766 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:32,766 INFO [RS:0;9514d6d5e5be:46353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46353 2024-11-09T21:13:32,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:13:32,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9514d6d5e5be,46353,1731186807216 2024-11-09T21:13:32,824 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:13:32,834 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9514d6d5e5be,46353,1731186807216] 2024-11-09T21:13:32,844 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9514d6d5e5be,46353,1731186807216 already deleted, retry=false 2024-11-09T21:13:32,845 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9514d6d5e5be,46353,1731186807216 on 9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:32,849 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:32,854 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 9514d6d5e5be,46353,1731186807216 (carryingMeta=true) 9514d6d5e5be,46353,1731186807216/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2b9d5c50[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-09T21:13:32,858 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:32,860 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true, isMeta: true 2024-11-09T21:13:32,862 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting 2024-11-09T21:13:32,864 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9514d6d5e5be,46353,1731186807216 WAL count=1, meta=true 2024-11-09T21:13:32,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta}] 2024-11-09T21:13:32,873 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:32,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:32,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:13:32,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:32,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46353-0x101217245da0001, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:32,935 INFO [RS:0;9514d6d5e5be:46353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:13:32,935 INFO [RS:0;9514d6d5e5be:46353 {}] regionserver.HRegionServer(1031): Exiting; stopping=9514d6d5e5be,46353,1731186807216; zookeeper connection closed. 2024-11-09T21:13:32,935 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d988e43 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d988e43 2024-11-09T21:13:33,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39113 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-09T21:13:33,065 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, size=31.4 K (32114bytes) 2024-11-09T21:13:33,066 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-09T21:13:33,066 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-09T21:13:33,066 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta 2024-11-09T21:13:33,072 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta after 4ms 2024-11-09T21:13:33,080 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta took 17ms 2024-11-09T21:13:33,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-09T21:13:33,096 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp 2024-11-09T21:13:33,100 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp 2024-11-09T21:13:33,101 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta so closing down 2024-11-09T21:13:33,101 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T21:13:33,105 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T21:13:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741864_1040 (size=31873) 2024-11-09T21:13:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741864_1040 (size=31873) 2024-11-09T21:13:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741864_1040 (size=31873) 2024-11-09T21:13:33,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp (wrote 57 edits, skipped 0 edits in 4 ms) 2024-11-09T21:13:33,132 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T21:13:33,135 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 50 ms; skipped=1; WAL=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-09T21:13:33,135 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, journal: Splitting hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, size=31.4 K (32114bytes) at 1731186813065Creating recovered edits writer path=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp at 1731186813100 (+35 ms)Finishing writing output for hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta so closing down at 1731186813101 (+1 ms)3 split writer threads finished at 1731186813105 (+4 ms)Closed recovered edits writer path=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp (wrote 57 edits, skipped 0 edits in 4 ms) at 1731186813129 (+24 ms)Rename recovered edits hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta.temp to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731186813132 (+3 ms)Processed 58 edits across 1 Regions in 50 ms; skipped=1; WAL=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731186813135 (+3 ms) 2024-11-09T21:13:33,135 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta 2024-11-09T21:13:33,137 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-09T21:13:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-09T21:13:33,145 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs 2024-11-09T21:13:33,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T21:13:33,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 in 271 msec 2024-11-09T21:13:33,150 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:33,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T21:13:33,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 in 284 msec 2024-11-09T21:13:33,156 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting dir is empty, no logs to split. 2024-11-09T21:13:33,156 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9514d6d5e5be,46353,1731186807216 WAL count=0, meta=true 2024-11-09T21:13:33,156 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9514d6d5e5be,46353,1731186807216 WAL splitting is done? wals=0, meta=true 2024-11-09T21:13:33,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T21:13:33,162 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:33,163 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:33,313 DEBUG [9514d6d5e5be:38717 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(204): Hosts are {9514d6d5e5be=0} racks are {/default-rack=0} 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T21:13:33,314 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T21:13:33,314 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T21:13:33,314 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T21:13:33,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:33,316 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPENING 2024-11-09T21:13:33,338 INFO [regionserver/9514d6d5e5be:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T21:13:33,339 INFO [regionserver/9514d6d5e5be:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T21:13:33,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,361 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:33,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:33,524 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T21:13:33,524 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:13:33,525 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T21:13:33,528 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C39113%2C1731186807372.meta, suffix=.meta, logDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372, archiveDir=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs, maxLogs=32 2024-11-09T21:13:33,545 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372/9514d6d5e5be%2C39113%2C1731186807372.meta.1731186813531.meta, exclude list is [], retry=0 2024-11-09T21:13:33,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-82bf6edd-a5c7-4747-981d-7c0c38502adf,DISK] 2024-11-09T21:13:33,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-9a1702e4-bdb6-4d2b-98cd-f725b6d0f2f2,DISK] 2024-11-09T21:13:33,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-95f5fa4d-f034-491f-82d9-b38aabc05eb1,DISK] 2024-11-09T21:13:33,551 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,39113,1731186807372/9514d6d5e5be%2C39113%2C1731186807372.meta.1731186813531.meta 2024-11-09T21:13:33,551 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41797:41797),(127.0.0.1/127.0.0.1:33715:33715),(127.0.0.1/127.0.0.1:46793:46793)] 2024-11-09T21:13:33,551 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T21:13:33,552 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T21:13:33,552 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T21:13:33,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:13:33,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:13:33,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:33,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:33,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:13:33,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:13:33,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:33,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:33,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:13:33,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:13:33,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:33,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:33,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:13:33,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:13:33,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:33,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:13:33,563 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:13:33,564 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:33,567 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740 2024-11-09T21:13:33,568 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T21:13:33,584 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T21:13:33,588 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-09T21:13:33,648 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/68b1e55b90ad47ca9d1041f683a354b4 is 138, key is test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004./info:regioninfo/1731186811463/Put/seqid=0 2024-11-09T21:13:33,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741866_1042 (size=34924) 2024-11-09T21:13:33,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741866_1042 (size=34924) 2024-11-09T21:13:33,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741866_1042 (size=34924) 2024-11-09T21:13:33,661 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/68b1e55b90ad47ca9d1041f683a354b4 2024-11-09T21:13:33,718 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/ns/6996e9a15d474ba4b643f4e67f607d4d is 43, key is default/ns:d/1731186810095/Put/seqid=0 2024-11-09T21:13:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741867_1043 (size=5153) 2024-11-09T21:13:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741867_1043 (size=5153) 2024-11-09T21:13:33,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741867_1043 (size=5153) 2024-11-09T21:13:33,726 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/ns/6996e9a15d474ba4b643f4e67f607d4d 2024-11-09T21:13:33,750 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/table/2fe8c587185e4870839d21e3cd469671 is 40, key is test/table:state/1731186811777/Put/seqid=0 2024-11-09T21:13:33,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741868_1044 (size=5165) 2024-11-09T21:13:33,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741868_1044 (size=5165) 2024-11-09T21:13:33,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741868_1044 (size=5165) 2024-11-09T21:13:33,759 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/table/2fe8c587185e4870839d21e3cd469671 2024-11-09T21:13:33,769 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/68b1e55b90ad47ca9d1041f683a354b4 as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/info/68b1e55b90ad47ca9d1041f683a354b4 2024-11-09T21:13:33,778 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/info/68b1e55b90ad47ca9d1041f683a354b4, entries=260, sequenceid=60, filesize=34.1 K 2024-11-09T21:13:33,781 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/ns/6996e9a15d474ba4b643f4e67f607d4d as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/ns/6996e9a15d474ba4b643f4e67f607d4d 2024-11-09T21:13:33,791 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/ns/6996e9a15d474ba4b643f4e67f607d4d, entries=2, sequenceid=60, filesize=5.0 K 2024-11-09T21:13:33,792 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/table/2fe8c587185e4870839d21e3cd469671 as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/table/2fe8c587185e4870839d21e3cd469671 2024-11-09T21:13:33,800 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/table/2fe8c587185e4870839d21e3cd469671, entries=2, sequenceid=60, filesize=5.0 K 2024-11-09T21:13:33,805 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 216ms, sequenceid=60, compaction requested=false; wal=null 2024-11-09T21:13:33,806 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T21:13:33,810 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T21:13:33,812 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:13:33,812 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:13:33,812 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:13:33,814 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:13:33,817 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-09T21:13:33,818 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60282334, jitterRate=-0.10172322392463684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:13:33,819 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T21:13:33,819 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731186813552Writing region info on filesystem at 1731186813552Initializing all the Stores at 1731186813554 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186813554Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186813554Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186813554Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186813554Obtaining lock to block concurrent updates at 1731186813588 (+34 ms)Preparing flush snapshotting stores in 1588230740 at 1731186813588Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731186813590 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731186813590Flushing 1588230740/info: creating writer at 1731186813592 (+2 ms)Flushing 1588230740/info: appending metadata at 1731186813638 (+46 ms)Flushing 1588230740/info: closing flushed file at 1731186813640 (+2 ms)Flushing 1588230740/ns: creating writer at 1731186813702 (+62 ms)Flushing 1588230740/ns: appending metadata at 1731186813717 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731186813717Flushing 1588230740/table: creating writer at 1731186813734 (+17 ms)Flushing 1588230740/table: appending metadata at 1731186813749 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731186813749Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e5b4c60: reopening flushed file at 1731186813767 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ee954b3: reopening flushed file at 1731186813779 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33259398: reopening flushed file at 1731186813791 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 216ms, sequenceid=60, compaction requested=false; wal=null at 1731186813805 (+14 ms)Cleaning up temporary data from old regions at 1731186813812 (+7 ms)Running coprocessor post-open hooks at 1731186813819 (+7 ms)Region opened successfully at 1731186813819 2024-11-09T21:13:33,820 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731186813517 2024-11-09T21:13:33,824 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T21:13:33,824 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T21:13:33,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:33,826 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:33,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:33,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:33,876 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:33,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T21:13:33,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 in 515 msec 2024-11-09T21:13:33,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-09T21:13:33,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 721 msec 2024-11-09T21:13:33,885 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 9514d6d5e5be,46353,1731186807216 had 9 regions 2024-11-09T21:13:33,886 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true, isMeta: false 2024-11-09T21:13:33,888 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9514d6d5e5be,46353,1731186807216 WAL count=1, meta=false 2024-11-09T21:13:33,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493}] 2024-11-09T21:13:33,890 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:33,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:34,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39113 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-09T21:13:34,070 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, size=2.2 K (2237bytes) 2024-11-09T21:13:34,070 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 2024-11-09T21:13:34,071 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 after 0ms 2024-11-09T21:13:34,074 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 took 4ms 2024-11-09T21:13:34,077 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4b29c44ccc5641890ba911d38607fb16: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,080 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 59b46e1f46651edf453a1e428f267809: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,082 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 66e1a5bbb9d115a37ba9c2ededd62632: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,083 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for af67ab20ba4ea57284753ee09fac2e5c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,085 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0d3eb0e0d62ca755868b2ca4e5785004: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,086 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2efa80434abe3c56dc2511eab9786a84: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,088 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 989c28b0ba07f1a68992722b1f5a8b3d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0e93a9320001bb729736b200e36ec3a0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 so closing down 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T21:13:34,090 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T21:13:34,090 INFO [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, journal: Splitting hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, size=2.2 K (2237bytes) at 1731186814070Finishing writing output for hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 so closing down at 1731186814090 (+20 ms)3 split writer threads finished at 1731186814090Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731186814090 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 2024-11-09T21:13:34,090 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T21:13:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T21:13:34,095 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting/9514d6d5e5be%2C46353%2C1731186807216.1731186809493 to hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs 2024-11-09T21:13:34,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T21:13:34,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 in 202 msec 2024-11-09T21:13:34,100 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-09T21:13:34,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 in 212 msec 2024-11-09T21:13:34,114 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/WALs/9514d6d5e5be,46353,1731186807216-splitting dir is empty, no logs to split. 2024-11-09T21:13:34,114 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9514d6d5e5be,46353,1731186807216 WAL count=0, meta=false 2024-11-09T21:13:34,114 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9514d6d5e5be,46353,1731186807216 WAL splitting is done? wals=0, meta=false 2024-11-09T21:13:34,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN}] 2024-11-09T21:13:34,119 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,120 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,121 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,122 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,124 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,125 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,126 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,127 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,127 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,128 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,129 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,130 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,132 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,133 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,135 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T21:13:34,271 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(204): Hosts are {9514d6d5e5be=0} racks are {/default-rack=0} 2024-11-09T21:13:34,271 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T21:13:34,271 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T21:13:34,271 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T21:13:34,271 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T21:13:34,271 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T21:13:34,271 INFO [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T21:13:34,272 DEBUG [9514d6d5e5be:38717 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T21:13:34,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,274 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9514d6d5e5be:46353 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:34,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, regionLocation=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-09T21:13:34,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 from cache 2024-11-09T21:13:34,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=59b46e1f46651edf453a1e428f267809, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,279 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:13:34,279 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1] 2024-11-09T21:13:34,279 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:13:34,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9514d6d5e5be:46353 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, regionLocation=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,281 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9514d6d5e5be:46353 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:34,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,46353,1731186807216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,39113,1731186807372, seqNum=-1, error=java.net.ConnectException: Call to address=9514d6d5e5be:46353 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9514d6d5e5be:46353 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9514d6d5e5be/172.17.0.2:46353 2024-11-09T21:13:34,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58349, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:13:34,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN because future has completed 2024-11-09T21:13:34,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:34,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN because future has completed 2024-11-09T21:13:34,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:34,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN because future has completed 2024-11-09T21:13:34,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:34,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN because future has completed 2024-11-09T21:13:34,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN because future has completed 2024-11-09T21:13:34,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN because future has completed 2024-11-09T21:13:34,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:34,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN because future has completed 2024-11-09T21:13:34,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN because future has completed 2024-11-09T21:13:34,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:34,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317}] 2024-11-09T21:13:34,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:34,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,39113,1731186807372}] 2024-11-09T21:13:34,452 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:34,452 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e93a9320001bb729736b200e36ec3a0, NAME => 'test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-09T21:13:34,452 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:34,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,453 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,455 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,456 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e93a9320001bb729736b200e36ec3a0 columnFamilyName family 2024-11-09T21:13:34,456 DEBUG [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:34,457 INFO [StoreOpener-0e93a9320001bb729736b200e36ec3a0-1 {}] regionserver.HStore(327): Store=0e93a9320001bb729736b200e36ec3a0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:34,457 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,458 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,460 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,461 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,461 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,463 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,464 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 0e93a9320001bb729736b200e36ec3a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60825089, jitterRate=-0.09363554418087006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:34,464 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:34,464 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 0e93a9320001bb729736b200e36ec3a0: Running coprocessor pre-open hook at 1731186814453Writing region info on filesystem at 1731186814453Initializing all the Stores at 1731186814454 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186814454Cleaning up temporary data from old regions at 1731186814461 (+7 ms)Running coprocessor post-open hooks at 1731186814464 (+3 ms)Region opened successfully at 1731186814464 2024-11-09T21:13:34,465 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0., pid=76, masterSystemTime=1731186814446 2024-11-09T21:13:34,467 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:34,467 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:34,468 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:34,468 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 59b46e1f46651edf453a1e428f267809, NAME => 'test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-09T21:13:34,468 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,468 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:34,468 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,469 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,470 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-09T21:13:34,470 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,470 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,471 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59b46e1f46651edf453a1e428f267809 columnFamilyName family 2024-11-09T21:13:34,471 DEBUG [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:34,470 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,472 INFO [StoreOpener-59b46e1f46651edf453a1e428f267809-1 {}] regionserver.HStore(327): Store=59b46e1f46651edf453a1e428f267809/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:34,472 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,473 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,474 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,475 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,475 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,477 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,478 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 59b46e1f46651edf453a1e428f267809; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68556823, jitterRate=0.02157627046108246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:34,478 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:34,478 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 59b46e1f46651edf453a1e428f267809: Running coprocessor pre-open hook at 1731186814469Writing region info on filesystem at 1731186814469Initializing all the Stores at 1731186814470 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186814470Cleaning up temporary data from old regions at 1731186814475 (+5 ms)Running coprocessor post-open hooks at 1731186814478 (+3 ms)Region opened successfully at 1731186814478 2024-11-09T21:13:34,479 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809., pid=72, masterSystemTime=1731186814446 2024-11-09T21:13:34,482 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:34,482 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:34,483 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:34,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b29c44ccc5641890ba911d38607fb16, NAME => 'test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-09T21:13:34,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:34,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,483 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,481 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-09T21:13:34,485 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,485 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,485 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,487 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b29c44ccc5641890ba911d38607fb16 columnFamilyName family 2024-11-09T21:13:34,487 DEBUG [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:34,487 INFO [StoreOpener-4b29c44ccc5641890ba911d38607fb16-1 {}] regionserver.HStore(327): Store=4b29c44ccc5641890ba911d38607fb16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:34,488 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,489 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,490 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,491 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,491 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,493 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,494 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 4b29c44ccc5641890ba911d38607fb16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63207289, jitterRate=-0.05813799798488617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:34,494 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:34,494 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 4b29c44ccc5641890ba911d38607fb16: Running coprocessor pre-open hook at 1731186814484Writing region info on filesystem at 1731186814484Initializing all the Stores at 1731186814485 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186814485Cleaning up temporary data from old regions at 1731186814491 (+6 ms)Running coprocessor post-open hooks at 1731186814494 (+3 ms)Region opened successfully at 1731186814494 2024-11-09T21:13:34,495 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16., pid=73, masterSystemTime=1731186814446 2024-11-09T21:13:34,497 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:34,497 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:34,497 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:34,497 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 2efa80434abe3c56dc2511eab9786a84, NAME => 'test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T21:13:34,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:34,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,498 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,500 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,501 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efa80434abe3c56dc2511eab9786a84 columnFamilyName family 2024-11-09T21:13:34,501 DEBUG [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:34,502 INFO [StoreOpener-2efa80434abe3c56dc2511eab9786a84-1 {}] regionserver.HStore(327): Store=2efa80434abe3c56dc2511eab9786a84/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:34,502 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,503 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,504 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,505 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,505 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,507 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,508 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 2efa80434abe3c56dc2511eab9786a84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67666570, jitterRate=0.008310467004776001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:34,508 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:34,508 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 2efa80434abe3c56dc2511eab9786a84: Running coprocessor pre-open hook at 1731186814498Writing region info on filesystem at 1731186814498Initializing all the Stores at 1731186814499 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186814499Cleaning up temporary data from old regions at 1731186814505 (+6 ms)Running coprocessor post-open hooks at 1731186814508 (+3 ms)Region opened successfully at 1731186814508 2024-11-09T21:13:34,509 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84., pid=77, masterSystemTime=1731186814446 2024-11-09T21:13:34,511 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:34,511 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:34,526 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,526 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,526 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,527 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,527 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,527 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,528 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,544 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,544 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,544 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,544 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure af67ab20ba4ea57284753ee09fac2e5c, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-11-09T21:13:34,548 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. 2024-11-09T21:13:34,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d3eb0e0d62ca755868b2ca4e5785004, NAME => 'test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-09T21:13:34,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,548 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:34,548 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,549 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,549 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,549 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,550 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,552 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3eb0e0d62ca755868b2ca4e5785004 columnFamilyName family 2024-11-09T21:13:34,552 DEBUG [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:34,552 INFO [StoreOpener-0d3eb0e0d62ca755868b2ca4e5785004-1 {}] regionserver.HStore(327): Store=0d3eb0e0d62ca755868b2ca4e5785004/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:34,552 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,553 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,555 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,556 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,556 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,558 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,559 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 0d3eb0e0d62ca755868b2ca4e5785004; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65228114, jitterRate=-0.02802535891532898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:34,559 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d3eb0e0d62ca755868b2ca4e5785004 2024-11-09T21:13:34,559 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 0d3eb0e0d62ca755868b2ca4e5785004: Running coprocessor pre-open hook at 1731186814549Writing region info on filesystem at 1731186814549Initializing all the Stores at 1731186814550 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186814550Cleaning up temporary data from old regions at 1731186814556 (+6 ms)Running coprocessor post-open hooks at 1731186814559 (+3 ms)Region opened successfully at 1731186814559 2024-11-09T21:13:34,560 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004., pid=75, masterSystemTime=1731186814542 2024-11-09T21:13:34,561 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844574 2024-11-09T21:13:34,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,574 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,579 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,585 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "9514d6d5e5be" port: 39113 start_code: 1731186807372 } transition { transition_code: OPENED region_info { region_id: 1731186810324 table_name { namespace: "default" qualifier: "test" } start_key: "bbb" end_key: "ccc" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1731186807532 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T21:13:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,612 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874612 2024-11-09T21:13:34,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934612 2024-11-09T21:13:34,612 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 37 msec 2024-11-09T21:13:34,612 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,612 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,612 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,612 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,612 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,613 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,613 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,616 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,617 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,621 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 989c28b0ba07f1a68992722b1f5a8b3d, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-09T21:13:34,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,628 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,628 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,635 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,635 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,635 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,647 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844647 2024-11-09T21:13:34,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,647 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,650 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,664 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,664 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874664 2024-11-09T21:13:34,664 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934664 2024-11-09T21:13:34,664 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 17 msec 2024-11-09T21:13:34,664 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,664 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,664 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,665 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,665 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,679 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,680 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,680 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,681 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efa80434abe3c56dc2511eab9786a84, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-09T21:13:34,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,683 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,683 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,696 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844697 2024-11-09T21:13:34,697 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,697 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,701 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,701 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,713 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,713 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874713 2024-11-09T21:13:34,713 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934713 2024-11-09T21:13:34,714 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-09T21:13:34,714 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,714 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,714 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,715 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,715 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,715 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,721 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,728 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-09T21:13:34,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,730 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,730 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844741 2024-11-09T21:13:34,741 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,741 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,742 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,744 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,752 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874752 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934752 2024-11-09T21:13:34,752 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,752 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,752 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,752 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,753 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,753 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,762 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d3eb0e0d62ca755868b2ca4e5785004, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-09T21:13:34,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,763 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,764 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,771 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,772 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844772 2024-11-09T21:13:34,772 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,772 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,774 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,775 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,775 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,775 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,783 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,783 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874783 2024-11-09T21:13:34,784 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934784 2024-11-09T21:13:34,784 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-09T21:13:34,784 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,784 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,784 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,784 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,784 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,784 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,784 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,784 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,789 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,792 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e1a5bbb9d115a37ba9c2ededd62632, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-09T21:13:34,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,794 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,794 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,803 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,804 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844804 2024-11-09T21:13:34,804 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,804 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,806 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,807 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,807 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,807 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,813 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,813 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874813 2024-11-09T21:13:34,813 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934813 2024-11-09T21:13:34,813 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:34,813 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,813 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,813 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,813 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,814 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,814 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,814 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,822 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,822 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b29c44ccc5641890ba911d38607fb16, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T21:13:34,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,823 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,823 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,825 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,825 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,835 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844835 2024-11-09T21:13:34,835 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,835 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,836 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,837 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,847 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874847 2024-11-09T21:13:34,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934847 2024-11-09T21:13:34,847 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-09T21:13:34,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,848 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,848 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b46e1f46651edf453a1e428f267809, server=9514d6d5e5be,37605,1731186807317 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-09T21:13:34,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,858 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,858 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,868 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844868 2024-11-09T21:13:34,868 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,868 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,870 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,870 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,871 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,877 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874877 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934877 2024-11-09T21:13:34,877 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,877 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,877 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,877 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,878 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,885 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN 2024-11-09T21:13:34,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T21:13:34,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=66e1a5bbb9d115a37ba9c2ededd62632, ASSIGN to state=OFFLINE, location=null, table=test, region=66e1a5bbb9d115a37ba9c2ededd62632 to restore RIT 2024-11-09T21:13:34,899 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844899 2024-11-09T21:13:34,899 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,899 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,900 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,901 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,907 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,907 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874907 2024-11-09T21:13:34,907 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934907 2024-11-09T21:13:34,907 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T21:13:34,907 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,907 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,907 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,908 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,908 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,908 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,909 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,913 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,914 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN 2024-11-09T21:13:34,914 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T21:13:34,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,915 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,915 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=989c28b0ba07f1a68992722b1f5a8b3d, ASSIGN to state=OFFLINE, location=null, table=test, region=989c28b0ba07f1a68992722b1f5a8b3d to restore RIT 2024-11-09T21:13:34,928 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844928 2024-11-09T21:13:34,928 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,928 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,929 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,930 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,931 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,938 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,938 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874938 2024-11-09T21:13:34,938 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934938 2024-11-09T21:13:34,938 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:34,938 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,938 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,938 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,939 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,939 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,939 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,945 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,945 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,945 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN 2024-11-09T21:13:34,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:34,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,946 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,946 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2efa80434abe3c56dc2511eab9786a84, ASSIGN to state=OFFLINE, location=null, table=test, region=2efa80434abe3c56dc2511eab9786a84 to restore RIT 2024-11-09T21:13:34,959 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844959 2024-11-09T21:13:34,959 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,960 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,961 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,963 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,969 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:34,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186874969 2024-11-09T21:13:34,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186934969 2024-11-09T21:13:34,969 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:34,969 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:34,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,969 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:34,969 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:34,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,970 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:34,970 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:34,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:34,970 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:34,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:34,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:34,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:34,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:34,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN 2024-11-09T21:13:34,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-09T21:13:34,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:34,979 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:34,979 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:34,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:34,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:34,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:34,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:34,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:34,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:34,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=59b46e1f46651edf453a1e428f267809, ASSIGN to state=OFFLINE, location=null, table=test, region=59b46e1f46651edf453a1e428f267809 to restore RIT 2024-11-09T21:13:34,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186844993 2024-11-09T21:13:34,993 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:34,994 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:34,996 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:34,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:34,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:34,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:34,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:34,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:34,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,002 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,002 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875002 2024-11-09T21:13:35,002 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935002 2024-11-09T21:13:35,002 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T21:13:35,002 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,003 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,003 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,003 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,003 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,003 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:35,011 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,011 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN 2024-11-09T21:13:35,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T21:13:35,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,012 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,012 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,024 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:35,024 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:35,025 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:35,025 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=af67ab20ba4ea57284753ee09fac2e5c, ASSIGN to state=OFFLINE, location=null, table=test, region=af67ab20ba4ea57284753ee09fac2e5c to restore RIT 2024-11-09T21:13:35,025 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845025 2024-11-09T21:13:35,025 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,025 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,029 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,029 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:35,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,029 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,036 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,036 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875036 2024-11-09T21:13:35,036 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935036 2024-11-09T21:13:35,036 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-09T21:13:35,036 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,036 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,036 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,037 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,037 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,037 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,037 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T21:13:35,039 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,040 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,044 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,044 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,044 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,044 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN 2024-11-09T21:13:35,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T21:13:35,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,045 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,045 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,056 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:35,056 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:35,056 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b29c44ccc5641890ba911d38607fb16, ASSIGN to state=OFFLINE, location=null, table=test, region=4b29c44ccc5641890ba911d38607fb16 to restore RIT 2024-11-09T21:13:35,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845057 2024-11-09T21:13:35,057 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,057 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,063 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,063 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,063 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,065 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875065 2024-11-09T21:13:35,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935065 2024-11-09T21:13:35,065 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T21:13:35,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,065 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,065 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,066 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,066 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,066 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,067 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,070 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,071 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,072 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,072 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,072 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,072 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN 2024-11-09T21:13:35,072 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T21:13:35,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,073 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,114 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:35,114 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e93a9320001bb729736b200e36ec3a0, ASSIGN to state=OFFLINE, location=null, table=test, region=0e93a9320001bb729736b200e36ec3a0 to restore RIT 2024-11-09T21:13:35,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845115 2024-11-09T21:13:35,115 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,115 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,117 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,118 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,122 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875122 2024-11-09T21:13:35,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935122 2024-11-09T21:13:35,122 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T21:13:35,122 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,122 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,122 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,123 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,123 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,123 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,125 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,125 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN 2024-11-09T21:13:35,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:35,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,129 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,129 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,143 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,144 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d3eb0e0d62ca755868b2ca4e5785004, ASSIGN to state=OFFLINE, location=null, table=test, region=0d3eb0e0d62ca755868b2ca4e5785004 to restore RIT 2024-11-09T21:13:35,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845151 2024-11-09T21:13:35,151 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,151 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,153 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,161 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875161 2024-11-09T21:13:35,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935161 2024-11-09T21:13:35,161 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:35,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,161 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,161 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,161 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,162 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:35,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,169 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,169 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,183 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845182 2024-11-09T21:13:35,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,183 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,185 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,186 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,186 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,187 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,191 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875192 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935192 2024-11-09T21:13:35,192 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,192 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,192 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,192 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,193 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,198 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.1731186809493, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:35,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,200 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,200 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845214 2024-11-09T21:13:35,215 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,215 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,218 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,218 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,218 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,225 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875225 2024-11-09T21:13:35,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935225 2024-11-09T21:13:35,225 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-09T21:13:35,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,225 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T21:13:35,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:35,232 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,232 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,245 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845245 2024-11-09T21:13:35,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,246 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,249 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,255 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875255 2024-11-09T21:13:35,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935255 2024-11-09T21:13:35,255 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T21:13:35,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,256 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,256 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,256 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T21:13:35,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,262 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:13:35,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,263 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,263 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845275 2024-11-09T21:13:35,275 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,275 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,276 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,282 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,282 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875282 2024-11-09T21:13:35,282 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935282 2024-11-09T21:13:35,282 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T21:13:35,283 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,283 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,283 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,283 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,283 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,283 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,283 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,283 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,284 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,286 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T21:13:35,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T21:13:35,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,288 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,299 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845299 2024-11-09T21:13:35,299 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,299 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,301 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,302 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,307 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875307 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935307 2024-11-09T21:13:35,307 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,307 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,307 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,307 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,307 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,308 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,308 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,310 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,310 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T21:13:35,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,311 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,311 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 9514d6d5e5be%2C46353%2C1731186807216.meta.1731186809931.meta, worker=9514d6d5e5be,39113,1731186807372 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T21:13:35,311 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T21:13:35,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,312 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,312 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,319 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845319 2024-11-09T21:13:35,319 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,319 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,321 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T21:13:35,322 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,327 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,327 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875327 2024-11-09T21:13:35,327 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935327 2024-11-09T21:13:35,327 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T21:13:35,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T21:13:35,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,327 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:35,327 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:35,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T21:13:35,328 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T21:13:35,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T21:13:35,328 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:13:35,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T21:13:35,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T21:13:35,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:13:35,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T21:13:35,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true 2024-11-09T21:13:35,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T21:13:35,331 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T21:13:35,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=61 2024-11-09T21:13:35,332 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9514d6d5e5be,39113,1731186807372, table=hbase:meta, region=1588230740 2024-11-09T21:13:35,332 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,39113,1731186807372, state=OPEN 2024-11-09T21:13:35,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:13:35,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:13:35,340 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186845340 2024-11-09T21:13:35,340 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:13:35,340 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:13:35,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T21:13:35,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 0e93a9320001bb729736b200e36ec3a0, server=9514d6d5e5be,37605,1731186807317:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9514d6d5e5be,46353,1731186807216, splitWal=true, meta=true exec-time=2.4970 sec 2024-11-09T21:13:35,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e306ecbfd9d382cf95e6deab029ad4fa, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d3eb0e0d62ca755868b2ca4e5785004, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6d1bef796951894c699e7a9348b65bd, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=974f175966e3c40bb61e40b6d82efdc2, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b29c44ccc5641890ba911d38607fb16, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcbe2f4ebe581c6c9b90065b0416620c, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b01a75f865260c525080606af78b0a41, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17cfda7d07986dd9effd1e28ad5cbc35, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=989c28b0ba07f1a68992722b1f5a8b3d, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e05a8d141de85a4320743bd44a2822ca, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff6038353628d35286993d84ec964b56, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=121419d26c305ff357f6891790ea3d8f, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b46e1f46651edf453a1e428f267809, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6999f5a07426b257e5d5577eb9b2495b, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00505fb66d17a1a78029487b077f81a5, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af67ab20ba4ea57284753ee09fac2e5c, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=397ac7d7817cb31ba8717b266a1ae0e9, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7301f6d77020efd526ad58dfb55d8407, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e93a9320001bb729736b200e36ec3a0, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9030947ba58303ad28b0aa1ef8091a57, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5007b602e53091a64151d33043eac752, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66e1a5bbb9d115a37ba9c2ededd62632, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=130debef05adf642209cee03bc162eda, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292dc5a8643963551c32ec0807b220c7, regionState=OPEN, lastHost=9514d6d5e5be,39113,1731186807372, regionLocation=9514d6d5e5be,39113,1731186807372, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2efa80434abe3c56dc2511eab9786a84, regionState=OPENING, lastHost=9514d6d5e5be,46353,1731186807216, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9fae39eda112797e02faba4a00be94f, regionState=OPEN, lastHost=9514d6d5e5be,37605,1731186807317, regionLocation=9514d6d5e5be,37605,1731186807317, openSeqNum=2 2024-11-09T21:13:35,348 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T21:13:35,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186875348 2024-11-09T21:13:35,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731186935348 2024-11-09T21:13:35,348 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T21:13:35,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 384), OpenFileDescriptor=618 (was 639), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=508 (was 535), ProcessCount=11 (was 11), AvailableMemoryMB=3623 (was 3659) 2024-11-09T21:13:35,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T21:13:35,365 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:13:35,365 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:13:35,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:13:35,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T21:13:35,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=385927850, stopped=false 2024-11-09T21:13:35,367 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9514d6d5e5be,38717,1731186806214 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:35,379 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:35,380 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:13:35,380 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:13:35,380 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:35,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,380 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:35,380 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:13:35,381 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9514d6d5e5be,37605,1731186807317' ***** 2024-11-09T21:13:35,381 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T21:13:35,381 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9514d6d5e5be,39113,1731186807372' ***** 2024-11-09T21:13:35,381 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T21:13:35,381 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T21:13:35,381 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T21:13:35,381 INFO [RS:1;9514d6d5e5be:37605 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T21:13:35,381 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T21:13:35,381 INFO [RS:1;9514d6d5e5be:37605 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T21:13:35,381 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T21:13:35,382 INFO [RS:2;9514d6d5e5be:39113 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T21:13:35,382 INFO [RS:2;9514d6d5e5be:39113 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T21:13:35,382 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for e05a8d141de85a4320743bd44a2822ca 2024-11-09T21:13:35,382 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e93a9320001bb729736b200e36ec3a0 2024-11-09T21:13:35,382 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for e306ecbfd9d382cf95e6deab029ad4fa 2024-11-09T21:13:35,382 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b29c44ccc5641890ba911d38607fb16 2024-11-09T21:13:35,382 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 5007b602e53091a64151d33043eac752 2024-11-09T21:13:35,382 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for b01a75f865260c525080606af78b0a41 2024-11-09T21:13:35,382 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 17cfda7d07986dd9effd1e28ad5cbc35 2024-11-09T21:13:35,382 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 974f175966e3c40bb61e40b6d82efdc2 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for b6d1bef796951894c699e7a9348b65bd 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 9030947ba58303ad28b0aa1ef8091a57 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 2efa80434abe3c56dc2511eab9786a84 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 7301f6d77020efd526ad58dfb55d8407 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 59b46e1f46651edf453a1e428f267809 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 121419d26c305ff357f6891790ea3d8f 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 6999f5a07426b257e5d5577eb9b2495b 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 00505fb66d17a1a78029487b077f81a5 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 397ac7d7817cb31ba8717b266a1ae0e9 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 292dc5a8643963551c32ec0807b220c7 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for ff6038353628d35286993d84ec964b56 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for 130debef05adf642209cee03bc162eda 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(3091): Received CLOSE for e9fae39eda112797e02faba4a00be94f 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(959): stopping server 9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(959): stopping server 9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e05a8d141de85a4320743bd44a2822ca, disabling compactions & flushes 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e93a9320001bb729736b200e36ec3a0, disabling compactions & flushes 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:13:35,383 INFO [RS:2;9514d6d5e5be:39113 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9514d6d5e5be:39113. 2024-11-09T21:13:35,383 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9514d6d5e5be:37605. 2024-11-09T21:13:35,383 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:35,383 DEBUG [RS:2;9514d6d5e5be:39113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:35,383 DEBUG [RS:1;9514d6d5e5be:37605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. after waiting 0 ms 2024-11-09T21:13:35,383 DEBUG [RS:2;9514d6d5e5be:39113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,383 DEBUG [RS:1;9514d6d5e5be:37605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. after waiting 0 ms 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:35,383 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:35,383 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-09T21:13:35,384 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T21:13:35,384 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T21:13:35,384 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T21:13:35,384 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T21:13:35,384 DEBUG [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1351): Waiting on 0e93a9320001bb729736b200e36ec3a0, 130debef05adf642209cee03bc162eda, 2efa80434abe3c56dc2511eab9786a84, 397ac7d7817cb31ba8717b266a1ae0e9, 4b29c44ccc5641890ba911d38607fb16, 59b46e1f46651edf453a1e428f267809, 6999f5a07426b257e5d5577eb9b2495b, 9030947ba58303ad28b0aa1ef8091a57, 974f175966e3c40bb61e40b6d82efdc2, b01a75f865260c525080606af78b0a41, e306ecbfd9d382cf95e6deab029ad4fa, e9fae39eda112797e02faba4a00be94f, ff6038353628d35286993d84ec964b56 2024-11-09T21:13:35,384 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-09T21:13:35,384 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1351): Waiting on 00505fb66d17a1a78029487b077f81a5, 0d3eb0e0d62ca755868b2ca4e5785004, 121419d26c305ff357f6891790ea3d8f, 1588230740, 17cfda7d07986dd9effd1e28ad5cbc35, 292dc5a8643963551c32ec0807b220c7, 5007b602e53091a64151d33043eac752, 7301f6d77020efd526ad58dfb55d8407, b6d1bef796951894c699e7a9348b65bd, e05a8d141de85a4320743bd44a2822ca, fcbe2f4ebe581c6c9b90065b0416620c 2024-11-09T21:13:35,385 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:13:35,385 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:13:35,385 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:13:35,385 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:13:35,385 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:13:35,385 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-09T21:13:35,389 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e05a8d141de85a4320743bd44a2822ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,389 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/0e93a9320001bb729736b200e36ec3a0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,391 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:35,391 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e93a9320001bb729736b200e36ec3a0: Waiting for close lock at 1731186815382Running coprocessor pre-close hooks at 1731186815383 (+1 ms)Disabling compacts and flushes for region at 1731186815383Disabling writes for close at 1731186815383Writing region close event to WAL at 1731186815384 (+1 ms)Running coprocessor post-close hooks at 1731186815390 (+6 ms)Closed at 1731186815391 (+1 ms) 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e05a8d141de85a4320743bd44a2822ca: Waiting for close lock at 1731186815382Running coprocessor pre-close hooks at 1731186815383 (+1 ms)Disabling compacts and flushes for region at 1731186815383Disabling writes for close at 1731186815383Writing region close event to WAL at 1731186815384 (+1 ms)Running coprocessor post-close hooks at 1731186815390 (+6 ms)Closed at 1731186815391 (+1 ms) 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731186810324.0e93a9320001bb729736b200e36ec3a0. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731186810324.e05a8d141de85a4320743bd44a2822ca. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e306ecbfd9d382cf95e6deab029ad4fa, disabling compactions & flushes 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5007b602e53091a64151d33043eac752, disabling compactions & flushes 2024-11-09T21:13:35,392 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:35,392 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. after waiting 0 ms 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731186810324.5007b602e53091a64151d33043eac752. after waiting 0 ms 2024-11-09T21:13:35,392 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:35,398 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e306ecbfd9d382cf95e6deab029ad4fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,398 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/5007b602e53091a64151d33043eac752/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,399 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:35,399 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5007b602e53091a64151d33043eac752: Waiting for close lock at 1731186815392Running coprocessor pre-close hooks at 1731186815392Disabling compacts and flushes for region at 1731186815392Disabling writes for close at 1731186815392Writing region close event to WAL at 1731186815393 (+1 ms)Running coprocessor post-close hooks at 1731186815399 (+6 ms)Closed at 1731186815399 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e306ecbfd9d382cf95e6deab029ad4fa: Waiting for close lock at 1731186815392Running coprocessor pre-close hooks at 1731186815392Disabling compacts and flushes for region at 1731186815392Disabling writes for close at 1731186815392Writing region close event to WAL at 1731186815393 (+1 ms)Running coprocessor post-close hooks at 1731186815398 (+5 ms)Closed at 1731186815399 (+1 ms) 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731186810324.5007b602e53091a64151d33043eac752. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731186810324.e306ecbfd9d382cf95e6deab029ad4fa. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17cfda7d07986dd9effd1e28ad5cbc35, disabling compactions & flushes 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b29c44ccc5641890ba911d38607fb16, disabling compactions & flushes 2024-11-09T21:13:35,399 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:35,399 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. after waiting 0 ms 2024-11-09T21:13:35,399 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. after waiting 0 ms 2024-11-09T21:13:35,400 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:35,400 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:35,405 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/17cfda7d07986dd9effd1e28ad5cbc35/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,405 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/882b7909ccd64cdfa9c7cbd55ff021d7 is 138, key is test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004./info:regioninfo/1731186814275/Put/seqid=0 2024-11-09T21:13:35,406 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:35,406 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17cfda7d07986dd9effd1e28ad5cbc35: Waiting for close lock at 1731186815399Running coprocessor pre-close hooks at 1731186815399Disabling compacts and flushes for region at 1731186815399Disabling writes for close at 1731186815399Writing region close event to WAL at 1731186815400 (+1 ms)Running coprocessor post-close hooks at 1731186815406 (+6 ms)Closed at 1731186815406 2024-11-09T21:13:35,406 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731186810324.17cfda7d07986dd9effd1e28ad5cbc35. 2024-11-09T21:13:35,406 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b6d1bef796951894c699e7a9348b65bd, disabling compactions & flushes 2024-11-09T21:13:35,406 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:35,406 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:35,407 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. after waiting 1 ms 2024-11-09T21:13:35,407 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:35,407 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/4b29c44ccc5641890ba911d38607fb16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,408 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b29c44ccc5641890ba911d38607fb16: Waiting for close lock at 1731186815399Running coprocessor pre-close hooks at 1731186815399Disabling compacts and flushes for region at 1731186815399Disabling writes for close at 1731186815399Writing region close event to WAL at 1731186815400 (+1 ms)Running coprocessor post-close hooks at 1731186815408 (+8 ms)Closed at 1731186815408 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731186810324.4b29c44ccc5641890ba911d38607fb16. 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b01a75f865260c525080606af78b0a41, disabling compactions & flushes 2024-11-09T21:13:35,409 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. after waiting 0 ms 2024-11-09T21:13:35,409 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:35,416 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b6d1bef796951894c699e7a9348b65bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,416 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/b01a75f865260c525080606af78b0a41/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,417 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b6d1bef796951894c699e7a9348b65bd: Waiting for close lock at 1731186815406Running coprocessor pre-close hooks at 1731186815406Disabling compacts and flushes for region at 1731186815406Disabling writes for close at 1731186815407 (+1 ms)Writing region close event to WAL at 1731186815408 (+1 ms)Running coprocessor post-close hooks at 1731186815416 (+8 ms)Closed at 1731186815416 2024-11-09T21:13:35,417 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731186810324.b6d1bef796951894c699e7a9348b65bd. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b01a75f865260c525080606af78b0a41: Waiting for close lock at 1731186815409Running coprocessor pre-close hooks at 1731186815409Disabling compacts and flushes for region at 1731186815409Disabling writes for close at 1731186815409Writing region close event to WAL at 1731186815411 (+2 ms)Running coprocessor post-close hooks at 1731186815417 (+6 ms)Closed at 1731186815417 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731186810324.b01a75f865260c525080606af78b0a41. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fcbe2f4ebe581c6c9b90065b0416620c, disabling compactions & flushes 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 974f175966e3c40bb61e40b6d82efdc2, disabling compactions & flushes 2024-11-09T21:13:35,417 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:35,417 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. after waiting 0 ms 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. after waiting 0 ms 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:35,417 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741869_1045 (size=8195) 2024-11-09T21:13:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741869_1045 (size=8195) 2024-11-09T21:13:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741869_1045 (size=8195) 2024-11-09T21:13:35,425 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/fcbe2f4ebe581c6c9b90065b0416620c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,426 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fcbe2f4ebe581c6c9b90065b0416620c: Waiting for close lock at 1731186815417Running coprocessor pre-close hooks at 1731186815417Disabling compacts and flushes for region at 1731186815417Disabling writes for close at 1731186815417Writing region close event to WAL at 1731186815420 (+3 ms)Running coprocessor post-close hooks at 1731186815426 (+6 ms)Closed at 1731186815426 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731186810324.fcbe2f4ebe581c6c9b90065b0416620c. 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7301f6d77020efd526ad58dfb55d8407, disabling compactions & flushes 2024-11-09T21:13:35,426 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. after waiting 0 ms 2024-11-09T21:13:35,426 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:35,429 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/974f175966e3c40bb61e40b6d82efdc2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,430 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 974f175966e3c40bb61e40b6d82efdc2: Waiting for close lock at 1731186815417Running coprocessor pre-close hooks at 1731186815417Disabling compacts and flushes for region at 1731186815417Disabling writes for close at 1731186815417Writing region close event to WAL at 1731186815420 (+3 ms)Running coprocessor post-close hooks at 1731186815430 (+10 ms)Closed at 1731186815430 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731186810324.974f175966e3c40bb61e40b6d82efdc2. 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9030947ba58303ad28b0aa1ef8091a57, disabling compactions & flushes 2024-11-09T21:13:35,430 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. after waiting 0 ms 2024-11-09T21:13:35,430 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:35,432 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/7301f6d77020efd526ad58dfb55d8407/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,432 INFO [regionserver/9514d6d5e5be:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:35,433 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:35,433 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7301f6d77020efd526ad58dfb55d8407: Waiting for close lock at 1731186815426Running coprocessor pre-close hooks at 1731186815426Disabling compacts and flushes for region at 1731186815426Disabling writes for close at 1731186815426Writing region close event to WAL at 1731186815428 (+2 ms)Running coprocessor post-close hooks at 1731186815433 (+5 ms)Closed at 1731186815433 2024-11-09T21:13:35,433 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731186810324.7301f6d77020efd526ad58dfb55d8407. 2024-11-09T21:13:35,433 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 121419d26c305ff357f6891790ea3d8f, disabling compactions & flushes 2024-11-09T21:13:35,433 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:35,433 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. after waiting 0 ms 2024-11-09T21:13:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:35,434 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/9030947ba58303ad28b0aa1ef8091a57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,435 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:35,435 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9030947ba58303ad28b0aa1ef8091a57: Waiting for close lock at 1731186815430Running coprocessor pre-close hooks at 1731186815430Disabling compacts and flushes for region at 1731186815430Disabling writes for close at 1731186815430Writing region close event to WAL at 1731186815431 (+1 ms)Running coprocessor post-close hooks at 1731186815435 (+4 ms)Closed at 1731186815435 2024-11-09T21:13:35,435 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731186810324.9030947ba58303ad28b0aa1ef8091a57. 2024-11-09T21:13:35,436 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2efa80434abe3c56dc2511eab9786a84, disabling compactions & flushes 2024-11-09T21:13:35,436 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:35,436 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:35,436 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. after waiting 0 ms 2024-11-09T21:13:35,436 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:35,438 INFO [regionserver/9514d6d5e5be:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:35,438 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/121419d26c305ff357f6891790ea3d8f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,439 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 121419d26c305ff357f6891790ea3d8f: Waiting for close lock at 1731186815433Running coprocessor pre-close hooks at 1731186815433Disabling compacts and flushes for region at 1731186815433Disabling writes for close at 1731186815434 (+1 ms)Writing region close event to WAL at 1731186815434Running coprocessor post-close hooks at 1731186815438 (+4 ms)Closed at 1731186815439 (+1 ms) 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731186810324.121419d26c305ff357f6891790ea3d8f. 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00505fb66d17a1a78029487b077f81a5, disabling compactions & flushes 2024-11-09T21:13:35,439 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. after waiting 0 ms 2024-11-09T21:13:35,439 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:35,440 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/2efa80434abe3c56dc2511eab9786a84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,441 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:35,441 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2efa80434abe3c56dc2511eab9786a84: Waiting for close lock at 1731186815436Running coprocessor pre-close hooks at 1731186815436Disabling compacts and flushes for region at 1731186815436Disabling writes for close at 1731186815436Writing region close event to WAL at 1731186815436Running coprocessor post-close hooks at 1731186815441 (+5 ms)Closed at 1731186815441 2024-11-09T21:13:35,441 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731186810324.2efa80434abe3c56dc2511eab9786a84. 2024-11-09T21:13:35,441 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 59b46e1f46651edf453a1e428f267809, disabling compactions & flushes 2024-11-09T21:13:35,441 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:35,442 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:35,442 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. after waiting 0 ms 2024-11-09T21:13:35,442 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:35,444 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/00505fb66d17a1a78029487b077f81a5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,444 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00505fb66d17a1a78029487b077f81a5: Waiting for close lock at 1731186815439Running coprocessor pre-close hooks at 1731186815439Disabling compacts and flushes for region at 1731186815439Disabling writes for close at 1731186815439Writing region close event to WAL at 1731186815440 (+1 ms)Running coprocessor post-close hooks at 1731186815444 (+4 ms)Closed at 1731186815444 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731186810324.00505fb66d17a1a78029487b077f81a5. 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 292dc5a8643963551c32ec0807b220c7, disabling compactions & flushes 2024-11-09T21:13:35,445 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. after waiting 0 ms 2024-11-09T21:13:35,445 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:35,447 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/59b46e1f46651edf453a1e428f267809/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,447 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 59b46e1f46651edf453a1e428f267809: Waiting for close lock at 1731186815441Running coprocessor pre-close hooks at 1731186815441Disabling compacts and flushes for region at 1731186815441Disabling writes for close at 1731186815442 (+1 ms)Writing region close event to WAL at 1731186815443 (+1 ms)Running coprocessor post-close hooks at 1731186815447 (+4 ms)Closed at 1731186815447 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731186810324.59b46e1f46651edf453a1e428f267809. 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6999f5a07426b257e5d5577eb9b2495b, disabling compactions & flushes 2024-11-09T21:13:35,448 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. after waiting 0 ms 2024-11-09T21:13:35,448 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:35,449 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/292dc5a8643963551c32ec0807b220c7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,450 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:35,450 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 292dc5a8643963551c32ec0807b220c7: Waiting for close lock at 1731186815445Running coprocessor pre-close hooks at 1731186815445Disabling compacts and flushes for region at 1731186815445Disabling writes for close at 1731186815445Writing region close event to WAL at 1731186815445Running coprocessor post-close hooks at 1731186815450 (+5 ms)Closed at 1731186815450 2024-11-09T21:13:35,450 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731186810324.292dc5a8643963551c32ec0807b220c7. 2024-11-09T21:13:35,452 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/6999f5a07426b257e5d5577eb9b2495b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,453 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6999f5a07426b257e5d5577eb9b2495b: Waiting for close lock at 1731186815448Running coprocessor pre-close hooks at 1731186815448Disabling compacts and flushes for region at 1731186815448Disabling writes for close at 1731186815448Writing region close event to WAL at 1731186815448Running coprocessor post-close hooks at 1731186815452 (+4 ms)Closed at 1731186815453 (+1 ms) 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731186810324.6999f5a07426b257e5d5577eb9b2495b. 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 397ac7d7817cb31ba8717b266a1ae0e9, disabling compactions & flushes 2024-11-09T21:13:35,453 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. after waiting 0 ms 2024-11-09T21:13:35,453 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:35,457 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/397ac7d7817cb31ba8717b266a1ae0e9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,457 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 397ac7d7817cb31ba8717b266a1ae0e9: Waiting for close lock at 1731186815453Running coprocessor pre-close hooks at 1731186815453Disabling compacts and flushes for region at 1731186815453Disabling writes for close at 1731186815453Writing region close event to WAL at 1731186815453Running coprocessor post-close hooks at 1731186815457 (+4 ms)Closed at 1731186815457 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731186810324.397ac7d7817cb31ba8717b266a1ae0e9. 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff6038353628d35286993d84ec964b56, disabling compactions & flushes 2024-11-09T21:13:35,458 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. after waiting 0 ms 2024-11-09T21:13:35,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:35,462 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/ff6038353628d35286993d84ec964b56/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,463 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff6038353628d35286993d84ec964b56: Waiting for close lock at 1731186815458Running coprocessor pre-close hooks at 1731186815458Disabling compacts and flushes for region at 1731186815458Disabling writes for close at 1731186815458Writing region close event to WAL at 1731186815458Running coprocessor post-close hooks at 1731186815463 (+5 ms)Closed at 1731186815463 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731186810324.ff6038353628d35286993d84ec964b56. 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 130debef05adf642209cee03bc162eda, disabling compactions & flushes 2024-11-09T21:13:35,463 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731186810324.130debef05adf642209cee03bc162eda. after waiting 0 ms 2024-11-09T21:13:35,463 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:35,467 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/130debef05adf642209cee03bc162eda/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,468 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 130debef05adf642209cee03bc162eda: Waiting for close lock at 1731186815463Running coprocessor pre-close hooks at 1731186815463Disabling compacts and flushes for region at 1731186815463Disabling writes for close at 1731186815463Writing region close event to WAL at 1731186815464 (+1 ms)Running coprocessor post-close hooks at 1731186815468 (+4 ms)Closed at 1731186815468 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731186810324.130debef05adf642209cee03bc162eda. 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e9fae39eda112797e02faba4a00be94f, disabling compactions & flushes 2024-11-09T21:13:35,468 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. after waiting 0 ms 2024-11-09T21:13:35,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:35,472 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/e9fae39eda112797e02faba4a00be94f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:13:35,473 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:35,473 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e9fae39eda112797e02faba4a00be94f: Waiting for close lock at 1731186815468Running coprocessor pre-close hooks at 1731186815468Disabling compacts and flushes for region at 1731186815468Disabling writes for close at 1731186815468Writing region close event to WAL at 1731186815469 (+1 ms)Running coprocessor post-close hooks at 1731186815472 (+3 ms)Closed at 1731186815473 (+1 ms) 2024-11-09T21:13:35,473 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731186810324.e9fae39eda112797e02faba4a00be94f. 2024-11-09T21:13:35,476 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:13:35,536 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-09T21:13:35,584 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(976): stopping server 9514d6d5e5be,37605,1731186807317; all regions closed. 2024-11-09T21:13:35,585 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1351): Waiting on 0d3eb0e0d62ca755868b2ca4e5785004, 1588230740 2024-11-09T21:13:35,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741833_1009 (size=7064) 2024-11-09T21:13:35,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741833_1009 (size=7064) 2024-11-09T21:13:35,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741833_1009 (size=7064) 2024-11-09T21:13:35,592 DEBUG [RS:1;9514d6d5e5be:37605 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/oldWALs 2024-11-09T21:13:35,592 INFO [RS:1;9514d6d5e5be:37605 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9514d6d5e5be%2C37605%2C1731186807317:(num 1731186809493) 2024-11-09T21:13:35,592 DEBUG [RS:1;9514d6d5e5be:37605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,592 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:35,592 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.ChoreService(370): Chore service for: regionserver/9514d6d5e5be:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T21:13:35,593 INFO [regionserver/9514d6d5e5be:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:13:35,593 INFO [RS:1;9514d6d5e5be:37605 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37605 2024-11-09T21:13:35,595 WARN [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004., aborting... java.io.IOException: Failed to report opened region to master: test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:35,596 ERROR [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server 9514d6d5e5be,39113,1731186807372: Failed to open region test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,bbb,1731186810324.0d3eb0e0d62ca755868b2ca4e5785004. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:35,596 ERROR [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-09T21:13:35,596 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-09T21:13:35,597 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-09T21:13:35,600 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-09T21:13:35,600 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-09T21:13:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9514d6d5e5be,37605,1731186807317 2024-11-09T21:13:35,602 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:13:35,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:13:35,606 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 475320576 }, "NonHeapMemoryUsage": { "committed": 162988032, "init": 7667712, "max": -1, "used": 160413624 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40669, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 27, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 13, "ProcessCallTime_75th_percentile": 20, "ProcessCallTime_90th_percentile": 24, "ProcessCallTime_95th_percentile": 25, "ProcessCallTime_98th_percentile": 26, "ProcessCallTime_99th_percentile": 26, "ProcessCallTime_99.9th_percentile": 26, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 27, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 13, "TotalCallTime_75th_percentile": 20, "TotalCallTime_90th_percentile": 24, "TotalCallTime_95th_percentile": 25, "TotalCallTime_98th_percentile": 26, "TotalCallTime_99th_percentile": 26, "TotalCallTime_99.9th_percentile": 26, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22855, "ResponseSize_mean": 4500, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17141, "ResponseSize_90th_percentile": 20569, "ResponseSize_95th_percentile": 21712, "ResponseSize_98th_percentile": 22397, "ResponseSize_99th_percentile": 22626, "ResponseSize_99.9th_percentile": 22832, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459921 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59919", "tag.serverName": "9514d6d5e5be,39113,1731186807372", "tag.clusterId": "5a6cc256-0d6e-4cf4-8d84-bd20115e44bf", "tag.Context": "regionserver", "tag.Hostname": "9514d6d5e5be", "regionCount": 1, "storeCount": 13, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 3, "maxStoreFileCount": 1, "memStoreSize": 0, "memStoreHeapSize": 3328, "memStoreOffHeapSize": 0, "storeFileSize": 45242, "storeFileSizeGrowthRate": 9048.4, "maxStoreFileAge": 621, "minStoreFileAge": 524, "avgStoreFileAge": 567, "numReferenceFiles": 0, "regionServerStartTime": 1731186807372, "averageRegionSize": 4524, "storeFileIndexSize": 1648, "staticIndexSize": 442, "staticBloomSize": 264, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922038576, "blockCacheCount": 4, "blockCacheDataBlockCount": 4, "blockCacheSize": 708304, "blockCacheCountHitPercent": 95.65217391304348, "blockCacheExpressHitPercent": 95.65217391304348, "l1CacheSize": 708304, "l1CacheFreeSize": 922038576, "l1CacheCount": 4, "l1CacheEvictionCount": 0, "l1CacheHitCount": 88, "l1CacheMissCount": 4, "l1CacheHitRatio": 0.9565217391304348, "l1CacheMissRatio": 0.043478260869565216, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 1530880, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 40, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 32, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 88, "blockCacheHitCountPrimary": 88, "blockCacheHitCachingCount": 88, "blockCacheMissCount": 4, "blockCacheMissCountPrimary": 4, "blockCacheMissCachingCount": 4, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 2, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 44, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 264, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 28672, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 32, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 32, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 2, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 45242, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 28672, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 45242, "FlushOutputSize_max": 45242, "FlushOutputSize_mean": 45242, "FlushOutputSize_25th_percentile": 45242, "FlushOutputSize_median": 45242, "FlushOutputSize_75th_percentile": 45242, "FlushOutputSize_90th_percentile": 45242, "FlushOutputSize_95th_percentile": 45242, "FlushOutputSize_98th_percentile": 45242, "FlushOutputSize_99th_percentile": 45242, "FlushOutputSize_99.9th_percentile": 45242, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 23, "ScanBlockBytesScanned_min": 28942, "ScanBlockBytesScanned_max": 28942, "ScanBlockBytesScanned_mean": 28942, "ScanBlockBytesScanned_25th_percentile": 28942, "ScanBlockBytesScanned_median": 28942, "ScanBlockBytesScanned_75th_percentile": 28942, "ScanBlockBytesScanned_90th_percentile": 28942, "ScanBlockBytesScanned_95th_percentile": 28942, "ScanBlockBytesScanned_98th_percentile": 28942, "ScanBlockBytesScanned_99th_percentile": 28942, "ScanBlockBytesScanned_99.9th_percentile": 28942, "ScanBlockBytesScanned_SizeRangeCount_0-10": 23, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 665666, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 28672, "FlushMemstoreSize_max": 28672, "FlushMemstoreSize_mean": 28672, "FlushMemstoreSize_25th_percentile": 28672, "FlushMemstoreSize_median": 28672, "FlushMemstoreSize_75th_percentile": 28672, "FlushMemstoreSize_90th_percentile": 28672, "FlushMemstoreSize_95th_percentile": 28672, "FlushMemstoreSize_98th_percentile": 28672, "FlushMemstoreSize_99th_percentile": 28672, "FlushMemstoreSize_99.9th_percentile": 28672, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 216, "FlushTime_max": 216, "FlushTime_mean": 216, "FlushTime_25th_percentile": 216, "FlushTime_median": 216, "FlushTime_75th_percentile": 216, "FlushTime_90th_percentile": 216, "FlushTime_95th_percentile": 216, "FlushTime_98th_percentile": 216, "FlushTime_99th_percentile": 216, "FlushTime_99.9th_percentile": 216, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 32, "ScanSize_min": 0, "ScanSize_max": 22728, "ScanSize_mean": 16335, "ScanSize_25th_percentile": 5682, "ScanSize_median": 11364, "ScanSize_75th_percentile": 17046, "ScanSize_90th_percentile": 20455, "ScanSize_95th_percentile": 21591, "ScanSize_98th_percentile": 22273, "ScanSize_99th_percentile": 22500, "ScanSize_99.9th_percentile": 22705, "ScanSize_SizeRangeCount_0-10": 32, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-09T21:13:35,607 WARN [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9514d6d5e5be:38717 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9514d6d5e5be:38717 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-09T21:13:35,608 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. 2024-11-09T21:13:35,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => af67ab20ba4ea57284753ee09fac2e5c, NAME => 'test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-09T21:13:35,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:35,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,608 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,610 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,611 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af67ab20ba4ea57284753ee09fac2e5c columnFamilyName family 2024-11-09T21:13:35,611 DEBUG [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:35,612 INFO [StoreOpener-af67ab20ba4ea57284753ee09fac2e5c-1 {}] regionserver.HStore(327): Store=af67ab20ba4ea57284753ee09fac2e5c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:35,612 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,612 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,613 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9514d6d5e5be,37605,1731186807317] 2024-11-09T21:13:35,614 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,614 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,615 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,616 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,617 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened af67ab20ba4ea57284753ee09fac2e5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69281770, jitterRate=0.032378822565078735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:35,617 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af67ab20ba4ea57284753ee09fac2e5c 2024-11-09T21:13:35,617 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for af67ab20ba4ea57284753ee09fac2e5c: Running coprocessor pre-open hook at 1731186815608Writing region info on filesystem at 1731186815608Initializing all the Stores at 1731186815609 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186815609Cleaning up temporary data from old regions at 1731186815615 (+6 ms)Running coprocessor post-open hooks at 1731186815617 (+2 ms)Region opened successfully at 1731186815617 2024-11-09T21:13:35,618 WARN [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9514d6d5e5be,39113,1731186807372 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ppp,1731186810324.af67ab20ba4ea57284753ee09fac2e5c. and can not recover 2024-11-09T21:13:35,619 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e1a5bbb9d115a37ba9c2ededd62632, NAME => 'test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,619 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,621 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,621 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e1a5bbb9d115a37ba9c2ededd62632 columnFamilyName family 2024-11-09T21:13:35,621 DEBUG [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:35,622 INFO [StoreOpener-66e1a5bbb9d115a37ba9c2ededd62632-1 {}] regionserver.HStore(327): Store=66e1a5bbb9d115a37ba9c2ededd62632/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:35,622 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,623 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,623 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9514d6d5e5be,37605,1731186807317 already deleted, retry=false 2024-11-09T21:13:35,623 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9514d6d5e5be,37605,1731186807317 expired; onlineServers=1 2024-11-09T21:13:35,624 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,625 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,625 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,627 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,627 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 66e1a5bbb9d115a37ba9c2ededd62632; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71713014, jitterRate=0.06860718131065369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:35,627 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e1a5bbb9d115a37ba9c2ededd62632 2024-11-09T21:13:35,628 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 66e1a5bbb9d115a37ba9c2ededd62632: Running coprocessor pre-open hook at 1731186815619Writing region info on filesystem at 1731186815619Initializing all the Stores at 1731186815620 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186815620Cleaning up temporary data from old regions at 1731186815625 (+5 ms)Running coprocessor post-open hooks at 1731186815628 (+3 ms)Region opened successfully at 1731186815628 2024-11-09T21:13:35,629 WARN [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9514d6d5e5be,39113,1731186807372 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:35,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,vvv,1731186810324.66e1a5bbb9d115a37ba9c2ededd62632. and can not recover 2024-11-09T21:13:35,629 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. 2024-11-09T21:13:35,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 989c28b0ba07f1a68992722b1f5a8b3d, NAME => 'test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-09T21:13:35,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:13:35,629 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,630 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,631 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,632 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 989c28b0ba07f1a68992722b1f5a8b3d columnFamilyName family 2024-11-09T21:13:35,632 DEBUG [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:13:35,633 INFO [StoreOpener-989c28b0ba07f1a68992722b1f5a8b3d-1 {}] regionserver.HStore(327): Store=989c28b0ba07f1a68992722b1f5a8b3d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:13:35,633 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,633 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,635 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/default/test/989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,635 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,635 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,637 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,638 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 989c28b0ba07f1a68992722b1f5a8b3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72171849, jitterRate=0.07544435560703278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:13:35,638 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 989c28b0ba07f1a68992722b1f5a8b3d 2024-11-09T21:13:35,638 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 989c28b0ba07f1a68992722b1f5a8b3d: Running coprocessor pre-open hook at 1731186815630Writing region info on filesystem at 1731186815630Initializing all the Stores at 1731186815631 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186815631Cleaning up temporary data from old regions at 1731186815635 (+4 ms)Running coprocessor post-open hooks at 1731186815638 (+3 ms)Region opened successfully at 1731186815638 2024-11-09T21:13:35,639 WARN [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9514d6d5e5be,39113,1731186807372 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:13:35,639 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,iii,1731186810324.989c28b0ba07f1a68992722b1f5a8b3d. and can not recover 2024-11-09T21:13:35,713 INFO [RS:1;9514d6d5e5be:37605 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:13:35,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:35,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37605-0x101217245da0002, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:35,713 INFO [RS:1;9514d6d5e5be:37605 {}] regionserver.HRegionServer(1031): Exiting; stopping=9514d6d5e5be,37605,1731186807317; zookeeper connection closed. 2024-11-09T21:13:35,713 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@70053b11 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@70053b11 2024-11-09T21:13:35,785 DEBUG [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:13:35,826 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/882b7909ccd64cdfa9c7cbd55ff021d7 2024-11-09T21:13:35,841 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/.tmp/info/882b7909ccd64cdfa9c7cbd55ff021d7 as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/info/882b7909ccd64cdfa9c7cbd55ff021d7 2024-11-09T21:13:35,851 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/info/882b7909ccd64cdfa9c7cbd55ff021d7, entries=24, sequenceid=71, filesize=8.0 K 2024-11-09T21:13:35,852 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 467ms, sequenceid=71, compaction requested=false 2024-11-09T21:13:35,857 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-09T21:13:35,858 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T21:13:35,858 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:13:35,858 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731186815384Running coprocessor pre-close hooks at 1731186815384Disabling compacts and flushes for region at 1731186815384Disabling writes for close at 1731186815385 (+1 ms)Obtaining lock to block concurrent updates at 1731186815385Preparing flush snapshotting stores in 1588230740 at 1731186815385Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731186815393 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731186815395 (+2 ms)Flushing 1588230740/info: creating writer at 1731186815397 (+2 ms)Flushing 1588230740/info: appending metadata at 1731186815404 (+7 ms)Flushing 1588230740/info: closing flushed file at 1731186815404Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d562fec: reopening flushed file at 1731186815840 (+436 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 467ms, sequenceid=71, compaction requested=false at 1731186815852 (+12 ms)Writing region close event to WAL at 1731186815854 (+2 ms)Running coprocessor post-close hooks at 1731186815858 (+4 ms)Closed at 1731186815858 2024-11-09T21:13:35,858 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T21:13:35,985 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(976): stopping server 9514d6d5e5be,39113,1731186807372; all regions closed. 2024-11-09T21:13:35,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741865_1041 (size=4204) 2024-11-09T21:13:35,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741865_1041 (size=4204) 2024-11-09T21:13:35,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741865_1041 (size=4204) 2024-11-09T21:13:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741834_1010 (size=5998) 2024-11-09T21:13:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741834_1010 (size=5998) 2024-11-09T21:13:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741834_1010 (size=5998) 2024-11-09T21:13:35,992 DEBUG [RS:2;9514d6d5e5be:39113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:13:35,992 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:13:35,992 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:13:35,992 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.ChoreService(370): Chore service for: regionserver/9514d6d5e5be:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T21:13:35,992 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:13:35,992 INFO [regionserver/9514d6d5e5be:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:13:35,992 INFO [RS:2;9514d6d5e5be:39113 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39113 2024-11-09T21:13:36,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:13:36,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9514d6d5e5be,39113,1731186807372 2024-11-09T21:13:36,055 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:13:36,066 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9514d6d5e5be,39113,1731186807372] 2024-11-09T21:13:36,076 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9514d6d5e5be,39113,1731186807372 already deleted, retry=false 2024-11-09T21:13:36,076 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9514d6d5e5be,39113,1731186807372 expired; onlineServers=0 2024-11-09T21:13:36,077 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9514d6d5e5be,38717,1731186806214' ***** 2024-11-09T21:13:36,077 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T21:13:36,077 INFO [M:0;9514d6d5e5be:38717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:13:36,078 INFO [M:0;9514d6d5e5be:38717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:13:36,078 DEBUG [M:0;9514d6d5e5be:38717 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T21:13:36,078 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T21:13:36,078 DEBUG [M:0;9514d6d5e5be:38717 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T21:13:36,079 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186809068 {}] cleaner.HFileCleaner(306): Exit Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186809068,5,FailOnTimeoutGroup] 2024-11-09T21:13:36,079 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186809067 {}] cleaner.HFileCleaner(306): Exit Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186809067,5,FailOnTimeoutGroup] 2024-11-09T21:13:36,079 INFO [M:0;9514d6d5e5be:38717 {}] hbase.ChoreService(370): Chore service for: master/9514d6d5e5be:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T21:13:36,080 INFO [M:0;9514d6d5e5be:38717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:13:36,080 DEBUG [M:0;9514d6d5e5be:38717 {}] master.HMaster(1795): Stopping service threads 2024-11-09T21:13:36,080 INFO [M:0;9514d6d5e5be:38717 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T21:13:36,081 INFO [M:0;9514d6d5e5be:38717 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:13:36,081 INFO [M:0;9514d6d5e5be:38717 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:13:36,082 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T21:13:36,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T21:13:36,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:13:36,087 DEBUG [M:0;9514d6d5e5be:38717 {}] zookeeper.ZKUtil(347): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T21:13:36,087 WARN [M:0;9514d6d5e5be:38717 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T21:13:36,088 INFO [M:0;9514d6d5e5be:38717 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/.lastflushedseqids 2024-11-09T21:13:36,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741870_1046 (size=1560) 2024-11-09T21:13:36,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741870_1046 (size=1560) 2024-11-09T21:13:36,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741870_1046 (size=1560) 2024-11-09T21:13:36,100 INFO [M:0;9514d6d5e5be:38717 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:13:36,101 INFO [M:0;9514d6d5e5be:38717 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T21:13:36,101 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:13:36,101 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:36,101 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:36,101 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:13:36,101 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:36,101 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.78 KB heapSize=302.30 KB 2024-11-09T21:13:36,117 DEBUG [M:0;9514d6d5e5be:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3918e02983404206ad14956befa598da is 82, key is hbase:meta,,1/info:regioninfo/1731186813824/Put/seqid=0 2024-11-09T21:13:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741871_1047 (size=6071) 2024-11-09T21:13:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741871_1047 (size=6071) 2024-11-09T21:13:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741871_1047 (size=6071) 2024-11-09T21:13:36,125 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3918e02983404206ad14956befa598da 2024-11-09T21:13:36,146 DEBUG [M:0;9514d6d5e5be:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b165e3cb15374bf990840f7c360d5d85 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731186815342/Put/seqid=0 2024-11-09T21:13:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741872_1048 (size=12801) 2024-11-09T21:13:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741872_1048 (size=12801) 2024-11-09T21:13:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741872_1048 (size=12801) 2024-11-09T21:13:36,153 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.50 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b165e3cb15374bf990840f7c360d5d85 2024-11-09T21:13:36,166 INFO [RS:2;9514d6d5e5be:39113 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:13:36,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:36,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39113-0x101217245da0003, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:36,166 INFO [RS:2;9514d6d5e5be:39113 {}] regionserver.HRegionServer(1031): Exiting; stopping=9514d6d5e5be,39113,1731186807372; zookeeper connection closed. 2024-11-09T21:13:36,166 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14984460 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14984460 2024-11-09T21:13:36,167 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-09T21:13:36,174 DEBUG [M:0;9514d6d5e5be:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72e415d83f0b471a8c7a9ae2bcc4b5ce is 69, key is 9514d6d5e5be,37605,1731186807317/rs:state/1731186809187/Put/seqid=0 2024-11-09T21:13:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741873_1049 (size=5442) 2024-11-09T21:13:36,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741873_1049 (size=5442) 2024-11-09T21:13:36,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741873_1049 (size=5442) 2024-11-09T21:13:36,181 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72e415d83f0b471a8c7a9ae2bcc4b5ce 2024-11-09T21:13:36,188 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 72e415d83f0b471a8c7a9ae2bcc4b5ce 2024-11-09T21:13:36,208 DEBUG [M:0;9514d6d5e5be:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/450b4a3b37b54ebd90131457ddd0352c is 52, key is load_balancer_on/state:d/1731186812683/Put/seqid=0 2024-11-09T21:13:36,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741874_1050 (size=5058) 2024-11-09T21:13:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741874_1050 (size=5058) 2024-11-09T21:13:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741874_1050 (size=5058) 2024-11-09T21:13:36,215 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/450b4a3b37b54ebd90131457ddd0352c 2024-11-09T21:13:36,222 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3918e02983404206ad14956befa598da as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3918e02983404206ad14956befa598da 2024-11-09T21:13:36,228 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3918e02983404206ad14956befa598da, entries=14, sequenceid=571, filesize=5.9 K 2024-11-09T21:13:36,229 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b165e3cb15374bf990840f7c360d5d85 as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b165e3cb15374bf990840f7c360d5d85 2024-11-09T21:13:36,235 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b165e3cb15374bf990840f7c360d5d85, entries=79, sequenceid=571, filesize=12.5 K 2024-11-09T21:13:36,237 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72e415d83f0b471a8c7a9ae2bcc4b5ce as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/72e415d83f0b471a8c7a9ae2bcc4b5ce 2024-11-09T21:13:36,244 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 72e415d83f0b471a8c7a9ae2bcc4b5ce 2024-11-09T21:13:36,244 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/72e415d83f0b471a8c7a9ae2bcc4b5ce, entries=3, sequenceid=571, filesize=5.3 K 2024-11-09T21:13:36,245 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/450b4a3b37b54ebd90131457ddd0352c as hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/450b4a3b37b54ebd90131457ddd0352c 2024-11-09T21:13:36,252 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35897/user/jenkins/test-data/cb870464-ca02-9a5b-82fc-a4e180bdbc05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/450b4a3b37b54ebd90131457ddd0352c, entries=1, sequenceid=571, filesize=4.9 K 2024-11-09T21:13:36,253 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.78 KB/254748, heapSize ~302.24 KB/309496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 152ms, sequenceid=571, compaction requested=false 2024-11-09T21:13:36,255 INFO [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:13:36,255 DEBUG [M:0;9514d6d5e5be:38717 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731186816101Disabling compacts and flushes for region at 1731186816101Disabling writes for close at 1731186816101Obtaining lock to block concurrent updates at 1731186816101Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731186816101Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254748, getHeapSize=309496, getOffHeapSize=0, getCellsCount=710 at 1731186816102 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731186816102Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731186816102Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731186816117 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731186816117Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731186816130 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731186816146 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731186816146Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731186816159 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731186816173 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731186816173Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731186816188 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731186816207 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731186816207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dfb3543: reopening flushed file at 1731186816221 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b7afeaa: reopening flushed file at 1731186816228 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24287e31: reopening flushed file at 1731186816236 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1de50763: reopening flushed file at 1731186816244 (+8 ms)Finished flush of dataSize ~248.78 KB/254748, heapSize ~302.24 KB/309496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 152ms, sequenceid=571, compaction requested=false at 1731186816253 (+9 ms)Writing region close event to WAL at 1731186816255 (+2 ms)Closed at 1731186816255 2024-11-09T21:13:36,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741830_1006 (size=292872) 2024-11-09T21:13:36,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741830_1006 (size=292872) 2024-11-09T21:13:36,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741830_1006 (size=292872) 2024-11-09T21:13:36,259 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:13:36,259 INFO [M:0;9514d6d5e5be:38717 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T21:13:36,259 INFO [M:0;9514d6d5e5be:38717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38717 2024-11-09T21:13:36,259 INFO [M:0;9514d6d5e5be:38717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:13:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:36,366 INFO [M:0;9514d6d5e5be:38717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:13:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x101217245da0000, quorum=127.0.0.1:59919, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:13:36,370 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:36,373 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:13:36,373 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:13:36,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:13:36,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,STOPPED} 2024-11-09T21:13:36,377 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T21:13:36,377 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T21:13:36,377 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1162922389-172.17.0.2-1731186801332 (Datanode Uuid 190569f7-1ad2-403a-b503-6bf63563d99c) service to localhost/127.0.0.1:35897 2024-11-09T21:13:36,377 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T21:13:36,378 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data5/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,378 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data6/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,379 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T21:13:36,381 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:36,382 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:13:36,382 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:13:36,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:13:36,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,STOPPED} 2024-11-09T21:13:36,383 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T21:13:36,383 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T21:13:36,383 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1162922389-172.17.0.2-1731186801332 (Datanode Uuid 411729ce-521a-432a-85e3-327a8e4035f0) service to localhost/127.0.0.1:35897 2024-11-09T21:13:36,383 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T21:13:36,384 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data3/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,384 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data4/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,385 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T21:13:36,393 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:13:36,393 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:13:36,393 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:13:36,394 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:13:36,394 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,STOPPED} 2024-11-09T21:13:36,395 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T21:13:36,395 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T21:13:36,395 WARN [BP-1162922389-172.17.0.2-1731186801332 heartbeating to localhost/127.0.0.1:35897 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1162922389-172.17.0.2-1731186801332 (Datanode Uuid aaa29a99-b9e0-4725-9a94-fe5cd18d3bc3) service to localhost/127.0.0.1:35897 2024-11-09T21:13:36,395 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T21:13:36,396 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data1/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,396 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/cluster_8afa236e-976c-2b63-3c6b-984b9e2f2f55/data/data2/current/BP-1162922389-172.17.0.2-1731186801332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:13:36,396 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T21:13:36,405 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:13:36,406 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:13:36,406 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:13:36,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:13:36,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/08ff5fff-7c02-5d4c-b718-f2946c6725c8/hadoop.log.dir/,STOPPED} 2024-11-09T21:13:36,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T21:13:36,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down